{
 "cells": [
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "%env LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1\n",
    "%env LLM_API_KEY=替换为自己的Qwen API Key"
   ],
   "id": "82878f99701ff63b"
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "af375836-b870-458b-87d1-4e00565977eb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:04:11.682880Z",
     "iopub.status.busy": "2024-12-14T09:04:11.682742Z",
     "iopub.status.idle": "2024-12-14T09:04:11.694381Z",
     "shell.execute_reply": "2024-12-14T09:04:11.693877Z",
     "shell.execute_reply.started": "2024-12-14T09:04:11.682867Z"
    },
    "papermill": {
     "duration": 0.115454,
     "end_time": "2024-11-23T14:29:00.919641",
     "exception": false,
     "start_time": "2024-11-23T14:29:00.804187",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%capture --no-stderr\n",
    "!pip install -U langchain langchain-community langchain-experimental langchain-openai pypdf sentence_transformers chromadb shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1e2c72b8-ee12-4130-af88-699998aa230c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:04:11.695045Z",
     "iopub.status.busy": "2024-12-14T09:04:11.694921Z",
     "iopub.status.idle": "2024-12-14T09:04:11.897377Z",
     "shell.execute_reply": "2024-12-14T09:04:11.896911Z",
     "shell.execute_reply.started": "2024-12-14T09:04:11.695033Z"
    },
    "papermill": {
     "duration": 0.319981,
     "end_time": "2024-11-23T14:29:01.380771",
     "exception": false,
     "start_time": "2024-11-23T14:29:01.060790",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "841d2b02-ad06-40d2-b11f-c7adccec6ca2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:04:11.898088Z",
     "iopub.status.busy": "2024-12-14T09:04:11.897917Z",
     "iopub.status.idle": "2024-12-14T09:04:12.006598Z",
     "shell.execute_reply": "2024-12-14T09:04:12.006156Z",
     "shell.execute_reply.started": "2024-12-14T09:04:11.898075Z"
    },
    "papermill": {
     "duration": 0.121409,
     "end_time": "2024-11-23T14:29:01.638126",
     "exception": false,
     "start_time": "2024-11-23T14:29:01.516717",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "expr_version = 'split_02_embedding_based_semantic_splitter'\n",
    "\n",
    "preprocess_output_dir = os.path.join(os.path.pardir, 'outputs', 'v1_20240713')\n",
    "qa_df = pd.read_excel(os.path.join(preprocess_output_dir, 'question_answer.xlsx'))\n",
    "expr_dir = os.path.join(os.path.pardir, 'experiments', expr_version)\n",
    "\n",
    "os.makedirs(expr_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf7e81e3-4c82-4842-aef5-7592caaf1d39",
   "metadata": {
    "papermill": {
     "duration": 0.100379,
     "end_time": "2024-11-23T14:29:01.862379",
     "exception": false,
     "start_time": "2024-11-23T14:29:01.762000",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 读取文档"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e6920e29-bc7d-4635-be06-d151eaf0e100",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:04:12.007335Z",
     "iopub.status.busy": "2024-12-14T09:04:12.007134Z",
     "iopub.status.idle": "2024-12-14T09:04:12.417348Z",
     "shell.execute_reply": "2024-12-14T09:04:12.416871Z",
     "shell.execute_reply.started": "2024-12-14T09:04:12.007322Z"
    },
    "papermill": {
     "duration": 2.012298,
     "end_time": "2024-11-23T14:29:03.974974",
     "exception": false,
     "start_time": "2024-11-23T14:29:01.962676",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": "markdown_document = open(os.path.join(os.path.pardir, 'outputs', 'MinerU_parsed_20241204', '2024全球经济金融展望报告.md')).read()"
  },
  {
   "cell_type": "markdown",
   "id": "841ec659-4ad7-4e1f-b1ea-3477bf97fde3",
   "metadata": {
    "papermill": {
     "duration": 0.100297,
     "end_time": "2024-11-23T14:29:04.219302",
     "exception": false,
     "start_time": "2024-11-23T14:29:04.119005",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 文档切分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2cddc9ac-88da-4c23-888d-481dc4a72f4a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:04:12.417949Z",
     "iopub.status.busy": "2024-12-14T09:04:12.417821Z",
     "iopub.status.idle": "2024-12-14T09:04:15.969727Z",
     "shell.execute_reply": "2024-12-14T09:04:15.969245Z",
     "shell.execute_reply.started": "2024-12-14T09:04:12.417936Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "device: cuda\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.10/site-packages/sentence_transformers/cross_encoder/CrossEncoder.py:11: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm, trange\n"
     ]
    }
   ],
   "source": [
    "from langchain.embeddings import HuggingFaceBgeEmbeddings\n",
    "import torch\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "print(f'device: {device}')\n",
    "\n",
    "def get_embeddings(model_path):\n",
    "    embeddings = HuggingFaceBgeEmbeddings(\n",
    "        model_name=model_path,\n",
    "        model_kwargs={'device': device},\n",
    "        encode_kwargs={'normalize_embeddings': True},\n",
    "        # show_progress=True\n",
    "        query_instruction='为这个句子生成表示以用于检索相关文章：'\n",
    "    )\n",
    "    return embeddings\n",
    "\n",
    "model_path = 'BAAI/bge-large-zh-v1.5'\n",
    "embeddings = get_embeddings(model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9865f891-22d4-4593-b9a9-97a265907f96",
   "metadata": {},
   "source": [
    "库中原始的切分正则表达式是为英文写的，中文无法正常切分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d6efb018-2090-4d89-9229-6459b7f392f2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:04:15.971554Z",
     "iopub.status.busy": "2024-12-14T09:04:15.971306Z",
     "iopub.status.idle": "2024-12-14T09:04:15.975931Z",
     "shell.execute_reply": "2024-12-14T09:04:15.975598Z",
     "shell.execute_reply.started": "2024-12-14T09:04:15.971541Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['太棒了\\n    我们继续吧。']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "re.split(r\"(?<=[.?!])\\s+\", '太棒了\\n    我们继续吧。')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "039be19c-fad7-4fc7-90fa-8172d46dc6a8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:04:15.976571Z",
     "iopub.status.busy": "2024-12-14T09:04:15.976340Z",
     "iopub.status.idle": "2024-12-14T09:04:15.990634Z",
     "shell.execute_reply": "2024-12-14T09:04:15.990300Z",
     "shell.execute_reply.started": "2024-12-14T09:04:15.976558Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['太棒了', '我们继续吧', '']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.split(r'[。！？\\n\\s]+', '太棒了\\n    我们继续吧。')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cbe6bc59-de9f-4d87-a2cb-badb3717e6ff",
   "metadata": {},
   "source": [
    "langchain_experimental的源码中提到，实现参考自：https://github.com/FullStackRetrieval-com/RetrievalTutorials/blob/main/tutorials/LevelsOfTextSplitting/5_Levels_Of_Text_Splitting.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "74fe856a-7c19-4c3c-bb30-7abfa6298f74",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:04:15.991300Z",
     "iopub.status.busy": "2024-12-14T09:04:15.991145Z",
     "iopub.status.idle": "2024-12-14T09:04:15.996816Z",
     "shell.execute_reply": "2024-12-14T09:04:15.996331Z",
     "shell.execute_reply.started": "2024-12-14T09:04:15.991288Z"
    },
    "papermill": {
     "duration": 0.109229,
     "end_time": "2024-11-23T14:29:04.429069",
     "exception": false,
     "start_time": "2024-11-23T14:29:04.319840",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "from langchain_experimental.text_splitter import SemanticChunker\n",
    "\n",
    "def split_with_embedding(embedding_model, text, breakpoint_threshold_type='percentile', number_of_chunks=None):\n",
    "    text_splitter = SemanticChunker(\n",
    "        embedding_model,\n",
    "        number_of_chunks=number_of_chunks,\n",
    "        # 此处设置能够切分中文的正则\n",
    "        sentence_split_regex=r\"[。！？\\n]+\",\n",
    "        breakpoint_threshold_type=breakpoint_threshold_type,\n",
    "        # 设置这个参数，避免分片过小\n",
    "        min_chunk_size=50\n",
    "    )\n",
    "    docs = text_splitter.create_documents([text])\n",
    "\n",
    "    return docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "37efeb76-6896-4fe0-913a-fd10a3ae0525",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:04:15.997462Z",
     "iopub.status.busy": "2024-12-14T09:04:15.997312Z",
     "iopub.status.idle": "2024-12-14T09:05:00.843964Z",
     "shell.execute_reply": "2024-12-14T09:05:00.843484Z",
     "shell.execute_reply.started": "2024-12-14T09:04:15.997450Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "breakpoint_threshold_types = [\"percentile\", \"standard_deviation\", \"interquartile\", \"gradient\"]\n",
    "\n",
    "splitted_docs_dict = {\n",
    "    f\"type_{key}\": split_with_embedding(embeddings, markdown_document, key) for key in breakpoint_threshold_types\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c226ad1b-318e-4f60-aa1c-7404e1d9f316",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:05:00.844589Z",
     "iopub.status.busy": "2024-12-14T09:05:00.844462Z",
     "iopub.status.idle": "2024-12-14T09:05:12.082557Z",
     "shell.execute_reply": "2024-12-14T09:05:12.082043Z",
     "shell.execute_reply.started": "2024-12-14T09:05:00.844577Z"
    }
   },
   "outputs": [],
   "source": [
    "splitted_docs_dict['clustering'] = split_with_embedding(embeddings, markdown_document, 'percentile', 120)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dff9e8ed-8c41-4004-8b52-c12e4e15e9c7",
   "metadata": {},
   "source": [
    "在后面的长度分布中可以看到，切分到的块普遍比较大，原始实现，作者在最后提到，可以对较大的块进行递归切分，此处我们只选一种类型，对它较大的块进行二次切分，并比较二次切分是\n",
    "否有效"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "924e348c-a552-460e-884f-4d94ca942e74",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:08:40.885351Z",
     "iopub.status.busy": "2024-12-14T09:08:40.884577Z",
     "iopub.status.idle": "2024-12-14T09:08:40.898371Z",
     "shell.execute_reply": "2024-12-14T09:08:40.896089Z",
     "shell.execute_reply.started": "2024-12-14T09:08:40.885278Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=700,\n",
    "    chunk_overlap=50,\n",
    "    separators=['\\n\\n', '\\n', '\\s', '，']\n",
    ")\n",
    "\n",
    "def further_split(docs, max_len, breakpoint_threshold_type):\n",
    "    ret_docs = []\n",
    "    for doc in docs:\n",
    "        if len(doc.page_content) >= 700:\n",
    "            ret_docs.extend(splitter.split_documents([doc]))\n",
    "        else:\n",
    "            ret_docs.append(doc)\n",
    "    return ret_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6cb99df1-c3e0-4eea-848c-a66ba56ff532",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:08:41.096774Z",
     "iopub.status.busy": "2024-12-14T09:08:41.096005Z",
     "iopub.status.idle": "2024-12-14T09:08:41.113381Z",
     "shell.execute_reply": "2024-12-14T09:08:41.111081Z",
     "shell.execute_reply.started": "2024-12-14T09:08:41.096702Z"
    }
   },
   "outputs": [],
   "source": [
    "splitted_docs_dict['type_percentile_further_split'] = further_split(splitted_docs_dict['type_percentile'], 700, 'percentile')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1652476c-3957-40fb-99ca-0a998390f094",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-13T03:00:55.978768Z",
     "iopub.status.busy": "2024-12-13T03:00:55.978064Z",
     "iopub.status.idle": "2024-12-13T03:00:55.987230Z",
     "shell.execute_reply": "2024-12-13T03:00:55.984859Z",
     "shell.execute_reply.started": "2024-12-13T03:00:55.978705Z"
    }
   },
   "source": [
    "## 检查一下切分后的块长度分布"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2adff9b0-96c9-496d-8c19-20a7232c2f75",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:08:44.432442Z",
     "iopub.status.busy": "2024-12-14T09:08:44.431672Z",
     "iopub.status.idle": "2024-12-14T09:08:44.480777Z",
     "shell.execute_reply": "2024-12-14T09:08:44.479547Z",
     "shell.execute_reply.started": "2024-12-14T09:08:44.432373Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=============== type_percentile ===============\n",
      "docs count: 41\n",
      "doc length distribution\n",
      "count      41.000000\n",
      "mean      821.658537\n",
      "std       806.092538\n",
      "min        50.000000\n",
      "25%       372.000000\n",
      "50%       576.000000\n",
      "75%      1007.000000\n",
      "90%      1739.000000\n",
      "97%      1882.800000\n",
      "99%      3602.000000\n",
      "max      4728.000000\n",
      "dtype: float64\n",
      "=============== type_standard_deviation ===============\n",
      "docs count: 15\n",
      "doc length distribution\n",
      "count       15.000000\n",
      "mean      2247.600000\n",
      "std       2912.896737\n",
      "min        100.000000\n",
      "25%        458.000000\n",
      "50%       1522.000000\n",
      "75%       2740.500000\n",
      "90%       3970.200000\n",
      "97%       8739.760000\n",
      "99%      10705.920000\n",
      "max      11689.000000\n",
      "dtype: float64\n",
      "=============== type_interquartile ===============\n",
      "docs count: 57\n",
      "doc length distribution\n",
      "count      57.000000\n",
      "mean      590.736842\n",
      "std       454.759337\n",
      "min        50.000000\n",
      "25%       276.000000\n",
      "50%       471.000000\n",
      "75%       738.000000\n",
      "90%      1259.200000\n",
      "97%      1673.040000\n",
      "99%      1815.560000\n",
      "max      1913.000000\n",
      "dtype: float64\n",
      "=============== type_gradient ===============\n",
      "docs count: 41\n",
      "doc length distribution\n",
      "count      41.000000\n",
      "mean      821.658537\n",
      "std       797.891146\n",
      "min       103.000000\n",
      "25%       324.000000\n",
      "50%       558.000000\n",
      "75%      1056.000000\n",
      "90%      1658.000000\n",
      "97%      2829.600000\n",
      "99%      3714.200000\n",
      "max      4159.000000\n",
      "dtype: float64\n",
      "=============== clustering ===============\n",
      "docs count: 97\n",
      "doc length distribution\n",
      "count     97.000000\n",
      "mean     346.721649\n",
      "std      188.393800\n",
      "min       50.000000\n",
      "25%      185.000000\n",
      "50%      326.000000\n",
      "75%      482.000000\n",
      "90%      571.800000\n",
      "97%      698.680000\n",
      "99%      905.520000\n",
      "max      990.000000\n",
      "dtype: float64\n",
      "=============== type_percentile_further_split ===============\n",
      "docs count: 72\n",
      "doc length distribution\n",
      "count     72.000000\n",
      "mean     480.236111\n",
      "std      220.043881\n",
      "min       50.000000\n",
      "25%      299.000000\n",
      "50%      540.000000\n",
      "75%      686.500000\n",
      "90%      692.000000\n",
      "97%      696.870000\n",
      "99%      697.000000\n",
      "max      697.000000\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "for key, splitted_docs in splitted_docs_dict.items():\n",
    "    print(f\"=============== {key} ===============\")\n",
    "    print(f\"docs count: {len(splitted_docs)}\")\n",
    "    print('doc length distribution')\n",
    "    print(pd.Series([len(d.page_content) for d in splitted_docs]).describe([0.25, 0.5, 0.75, 0.9, 0.97, 0.99]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "220dbc3a-fceb-4e49-a3f1-01e16660b2a6",
   "metadata": {
    "papermill": {
     "duration": 0.100209,
     "end_time": "2024-11-23T14:29:05.255871",
     "exception": false,
     "start_time": "2024-11-23T14:29:05.155662",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f6f46c73-7369-448f-a89a-ed3d817cad47",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:08:58.084205Z",
     "iopub.status.busy": "2024-12-14T09:08:58.083415Z",
     "iopub.status.idle": "2024-12-14T09:08:58.109711Z",
     "shell.execute_reply": "2024-12-14T09:08:58.109233Z",
     "shell.execute_reply.started": "2024-12-14T09:08:58.084134Z"
    },
    "papermill": {
     "duration": 83.983138,
     "end_time": "2024-11-23T14:35:06.117207",
     "exception": false,
     "start_time": "2024-11-23T14:33:42.134069",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import jieba\n",
    "import shutil\n",
    "\n",
    "from tqdm.auto import tqdm\n",
    "from langchain_community.vectorstores import Chroma\n",
    "\n",
    "def get_vector_db(splitted_docs, embeddings, name):\n",
    "    persist_directory = os.path.join(expr_dir, 'chroma', 'bge', name)\n",
    "    # persist_directory = os.path.join('/tmp', 'chroma', 'bge', name)\n",
    "    shutil.rmtree(persist_directory, ignore_errors=True)\n",
    "    vector_db = Chroma.from_documents(\n",
    "        splitted_docs,\n",
    "        embedding=embeddings,\n",
    "        persist_directory=persist_directory\n",
    "    )\n",
    "    return vector_db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "11e148bd-7a3f-4bde-a574-21a30f337542",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:08:58.333036Z",
     "iopub.status.busy": "2024-12-14T09:08:58.332248Z",
     "iopub.status.idle": "2024-12-14T09:10:33.796467Z",
     "shell.execute_reply": "2024-12-14T09:10:33.795515Z",
     "shell.execute_reply.started": "2024-12-14T09:08:58.332963Z"
    }
   },
   "outputs": [],
   "source": [
    "vector_db_dict = {\n",
    "    key: get_vector_db(splitted_docs, embeddings, f\"{key}_chroma\")\n",
    "    for key, splitted_docs in splitted_docs_dict.items()\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7925564a-7d30-4914-baaf-4a00abb7686d",
   "metadata": {
    "papermill": {
     "duration": 0.109216,
     "end_time": "2024-11-23T14:35:26.464009",
     "exception": false,
     "start_time": "2024-11-23T14:35:26.354793",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 生成答案"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92c68613-44eb-400b-b869-2e3fd3446c35",
   "metadata": {},
   "source": [
    "此处的qwen2:7b-instruct-32k是经过上下文扩充后的qwen2:7b-instruct模型，具体方法可以参考：https://mp.weixin.qq.com/s/n0RLhQNcWRPKNBJwaX-a2g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "27132c3b-0051-4df6-bf57-fd804acb8d17",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:10:33.798375Z",
     "iopub.status.busy": "2024-12-14T09:10:33.798057Z",
     "iopub.status.idle": "2024-12-14T09:10:33.940679Z",
     "shell.execute_reply": "2024-12-14T09:10:33.940189Z",
     "shell.execute_reply.started": "2024-12-14T09:10:33.798345Z"
    },
    "papermill": {
     "duration": 0.199165,
     "end_time": "2024-11-23T14:35:27.323500",
     "exception": false,
     "start_time": "2024-11-23T14:35:27.124335",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3334015/2053630348.py:3: LangChainDeprecationWarning: The class `Ollama` was deprecated in LangChain 0.3.1 and will be removed in 1.0.0. An updated version of the class exists in the :class:`~langchain-ollama package and should be used instead. To use it run `pip install -U :class:`~langchain-ollama` and import as `from :class:`~langchain_ollama import OllamaLLM``.\n",
      "  ollama_llm = Ollama(\n"
     ]
    }
   ],
   "source": [
    "from langchain.llms import Ollama\n",
    "\n",
    "ollama_llm = Ollama(\n",
    "    model='qwen2:7b-instruct-32k',\n",
    "    base_url='http://localhost:11434',\n",
    "    top_k=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "50404beb-3be0-4aaa-b124-8c7a52b84531",
   "metadata": {
    "editable": true,
    "execution": {
     "iopub.execute_input": "2024-12-14T09:10:33.941418Z",
     "iopub.status.busy": "2024-12-14T09:10:33.941136Z",
     "iopub.status.idle": "2024-12-14T09:10:33.944752Z",
     "shell.execute_reply": "2024-12-14T09:10:33.944430Z",
     "shell.execute_reply.started": "2024-12-14T09:10:33.941405Z"
    },
    "papermill": {
     "duration": 0.159318,
     "end_time": "2024-11-23T14:35:26.768506",
     "exception": false,
     "start_time": "2024-11-23T14:35:26.609188",
     "status": "completed"
    },
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "def rag(vector_db, llm, query, n_chunks=4):\n",
    "    prompt_tmpl = \"\"\"\n",
    "你是一个金融分析师，擅长根据所获取的信息片段，对问题进行分析和推理。\n",
    "你的任务是根据所获取的信息片段（<<<<context>>><<<</context>>>之间的内容）回答问题。\n",
    "回答保持简洁，不必重复问题，不要添加描述性解释和与答案无关的任何内容。\n",
    "已知信息：\n",
    "<<<<context>>>\n",
    "{{knowledge}}\n",
    "<<<</context>>>\n",
    "\n",
    "问题：{{query}}\n",
    "请回答：\n",
    "\"\"\".strip()\n",
    "    chunks = vector_db.similarity_search(query, k=n_chunks)\n",
    "    prompt = prompt_tmpl.replace('{{knowledge}}', '\\n\\n'.join([doc.page_content for doc in chunks])).replace('{{query}}', query)\n",
    "    retry_count = 3\n",
    "\n",
    "    resp = ''\n",
    "    while retry_count > 0:\n",
    "        try:\n",
    "            resp = llm.invoke(prompt)\n",
    "            break\n",
    "        except Exception as e:\n",
    "            retry_count -= 1\n",
    "            sleeping_seconds = 2 ** (4 - retry_count)\n",
    "            print(f\"query={query}, error={e}, sleeping={sleeping_seconds}, remaining retry count={retry_count}\")\n",
    "            \n",
    "            time.sleep(sleeping_seconds)\n",
    "    \n",
    "    return resp, chunks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95e5a804-2dc6-411c-ba71-6ccf765b2b73",
   "metadata": {
    "papermill": {
     "duration": 0.135973,
     "end_time": "2024-11-23T14:35:27.001401",
     "exception": false,
     "start_time": "2024-11-23T14:35:26.865428",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "166392d8-f801-4372-b8ad-3e79aef0b350",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:10:33.945686Z",
     "iopub.status.busy": "2024-12-14T09:10:33.945565Z",
     "iopub.status.idle": "2024-12-14T09:10:33.959785Z",
     "shell.execute_reply": "2024-12-14T09:10:33.959338Z",
     "shell.execute_reply.started": "2024-12-14T09:10:33.945675Z"
    },
    "papermill": {
     "duration": 0.141864,
     "end_time": "2024-11-23T14:35:27.564409",
     "exception": false,
     "start_time": "2024-11-23T14:35:27.422545",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "prediction_df = qa_df[qa_df['dataset'] == 'test'][['uuid', 'question', 'qa_type', 'answer']].rename(columns={'answer': 'ref_answer'})\n",
    "\n",
    "def predict(vector_db, llm, prediction_df, n_chunks):\n",
    "    prediction_df = prediction_df.copy()\n",
    "    answer_dict = {}\n",
    "\n",
    "    for idx, row in tqdm(prediction_df.iterrows(), total=len(prediction_df)):\n",
    "        uuid = row['uuid']\n",
    "        question = row['question']\n",
    "        answer, chunks = rag(vector_db, llm, question, n_chunks=n_chunks)\n",
    "        assert len(chunks) <= n_chunks\n",
    "        answer_dict[question] = {\n",
    "            'uuid': uuid,\n",
    "            'ref_answer': row['ref_answer'],\n",
    "            'gen_answer': answer,\n",
    "            'chunks': chunks\n",
    "        }\n",
    "\n",
    "    prediction_df.loc[:, 'gen_answer'] = prediction_df['question'].apply(lambda q: answer_dict[q]['gen_answer'])\n",
    "    prediction_df.loc[:, 'chunks'] = prediction_df['question'].apply(lambda q: answer_dict[q]['chunks'])\n",
    "\n",
    "    return prediction_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f881aa9c-bf51-4120-95ca-9108b53e48cb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:10:33.960325Z",
     "iopub.status.busy": "2024-12-14T09:10:33.960200Z",
     "iopub.status.idle": "2024-12-14T09:10:33.966394Z",
     "shell.execute_reply": "2024-12-14T09:10:33.966012Z",
     "shell.execute_reply.started": "2024-12-14T09:10:33.960313Z"
    }
   },
   "outputs": [],
   "source": [
    "save_path = os.path.join(expr_dir, 'preds.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f40b1241-fb9a-43dd-a755-bf9409e90be2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:10:33.967018Z",
     "iopub.status.busy": "2024-12-14T09:10:33.966863Z",
     "iopub.status.idle": "2024-12-14T09:10:33.972137Z",
     "shell.execute_reply": "2024-12-14T09:10:33.971810Z",
     "shell.execute_reply.started": "2024-12-14T09:10:33.967006Z"
    }
   },
   "outputs": [],
   "source": [
    "if os.path.exists(save_path):\n",
    "    print(f'found cache at {save_path}')\n",
    "    pred_dict = pickle.load(open(save_path, 'rb'))\n",
    "else:\n",
    "    pred_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "5b3dff59-03e8-4343-b12f-66c16b4bfa33",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T09:10:33.972688Z",
     "iopub.status.busy": "2024-12-14T09:10:33.972539Z",
     "iopub.status.idle": "2024-12-14T10:02:37.468977Z",
     "shell.execute_reply": "2024-12-14T10:02:37.466625Z",
     "shell.execute_reply.started": "2024-12-14T09:10:33.972676Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediting for type_percentile ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85346b4fdd8a483db32428b73c81958f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediting for type_standard_deviation ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4fd8f1c0183c41ecb2283ba90b008935",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediting for type_interquartile ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e37e94d38a534cdb93bfbf90f338fd5b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediting for type_gradient ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b515f8ed356d4b55a33b5ff9f8e6b92c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediting for clustering ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1083d74a3765457c9f9ee79f23c3ca18",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediting for type_percentile_further_split ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "768c627bc5614f9885aac824da40121f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for key, vector_db in vector_db_dict.items():\n",
    "    print(f'prediting for {key} ...')\n",
    "    if key in pred_dict:\n",
    "        continue\n",
    "    pred_df = predict(vector_db, ollama_llm, prediction_df, n_chunks=3)\n",
    "    pred_dict[key] = pred_df\n",
    "    pickle.dump(pred_dict, open(save_path, 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d79e974-089f-4c08-ba5e-804f6542e06a",
   "metadata": {
    "papermill": {
     "duration": 0.14423,
     "end_time": "2024-11-23T14:44:03.513124",
     "exception": false,
     "start_time": "2024-11-23T14:44:03.368894",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "217568fe-c0e4-49eb-9a7c-9fdfbc033d8a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T10:02:37.472509Z",
     "iopub.status.busy": "2024-12-14T10:02:37.471790Z",
     "iopub.status.idle": "2024-12-14T10:02:37.779573Z",
     "shell.execute_reply": "2024-12-14T10:02:37.779080Z",
     "shell.execute_reply.started": "2024-12-14T10:02:37.472440Z"
    },
    "papermill": {
     "duration": 0.369729,
     "end_time": "2024-11-23T14:44:04.017198",
     "exception": false,
     "start_time": "2024-11-23T14:44:03.647469",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain_openai import ChatOpenAI\n",
    "import time\n",
    "\n",
    "judge_llm = ChatOpenAI(\n",
    "    api_key=os.environ['LLM_API_KEY'],\n",
    "    base_url=os.environ['LLM_BASE_URL'],\n",
    "    model_name='qwen2-72b-instruct',\n",
    "    temperature=0\n",
    ")\n",
    "\n",
    "def evaluate(prediction_df):\n",
    "    \"\"\"\n",
    "    对预测结果进行打分\n",
    "    :param prediction_df: 预测结果，需要包含问题，参考答案，生成的答案，列名分别为question, ref_answer, gen_answer\n",
    "    :return 打分模型原始返回结果\n",
    "    \"\"\"\n",
    "    prompt_tmpl = \"\"\"\n",
    "你是一个经济学博士，现在我有一系列问题，有一个助手已经对这些问题进行了回答，你需要参照参考答案，评价这个助手的回答是否正确，仅回复“是”或“否”即可，不要带其他描述性内容或无关信息。\n",
    "问题：\n",
    "<question>\n",
    "{{question}}\n",
    "</question>\n",
    "\n",
    "参考答案：\n",
    "<ref_answer>\n",
    "{{ref_answer}}\n",
    "</ref_answer>\n",
    "\n",
    "助手回答：\n",
    "<gen_answer>\n",
    "{{gen_answer}}\n",
    "</gen_answer>\n",
    "请评价：\n",
    "    \"\"\"\n",
    "    results = []\n",
    "\n",
    "    for _, row in tqdm(prediction_df.iterrows(), total=len(prediction_df)):\n",
    "        question = row['question']\n",
    "        ref_answer = row['ref_answer']\n",
    "        gen_answer = row['gen_answer']\n",
    "\n",
    "        prompt = prompt_tmpl.replace('{{question}}', question).replace('{{ref_answer}}', str(ref_answer)).replace('{{gen_answer}}', gen_answer).strip()\n",
    "        \n",
    "        retry_count = 3\n",
    "        result = ''\n",
    "        \n",
    "        while retry_count > 0:\n",
    "            try:\n",
    "                result = judge_llm.invoke(prompt).content\n",
    "                break\n",
    "            except Exception as e:\n",
    "                retry_count -= 1\n",
    "                sleeping_seconds = 2 ** (4 - retry_count)\n",
    "                print(f\"query={question}, error={e}, sleeping={sleeping_seconds}, remaining retry count={retry_count}\")\n",
    "                \n",
    "                time.sleep(sleeping_seconds)\n",
    "        \n",
    "        results.append(result)\n",
    "\n",
    "        time.sleep(1)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "fa6420f8-2d67-43bd-8ec6-2f136701e943",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T10:02:37.780357Z",
     "iopub.status.busy": "2024-12-14T10:02:37.780064Z",
     "iopub.status.idle": "2024-12-14T10:02:37.782577Z",
     "shell.execute_reply": "2024-12-14T10:02:37.782263Z",
     "shell.execute_reply.started": "2024-12-14T10:02:37.780344Z"
    }
   },
   "outputs": [],
   "source": [
    "eval_save_path = os.path.join(expr_dir, 'evals.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ed65b04c-0b2a-49ae-9892-199f4ca8ee36",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T10:02:37.783878Z",
     "iopub.status.busy": "2024-12-14T10:02:37.783749Z",
     "iopub.status.idle": "2024-12-14T10:02:37.795944Z",
     "shell.execute_reply": "2024-12-14T10:02:37.795519Z",
     "shell.execute_reply.started": "2024-12-14T10:02:37.783865Z"
    }
   },
   "outputs": [],
   "source": [
    "if os.path.exists(eval_save_path):\n",
    "    print(f\"found cache at {eval_save_path}\")\n",
    "    eval_dict = pickle.load(open(eval_save_path, 'rb'))\n",
    "else:\n",
    "    eval_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d1e2d6c9-c45f-442b-8181-66c30437a308",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T10:02:37.796459Z",
     "iopub.status.busy": "2024-12-14T10:02:37.796337Z",
     "iopub.status.idle": "2024-12-14T10:17:53.347347Z",
     "shell.execute_reply": "2024-12-14T10:17:53.344915Z",
     "shell.execute_reply.started": "2024-12-14T10:02:37.796447Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "evaluating for type_percentile ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2bed2bc97c13402dafaac14bb5d44a40",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key=type_percentile raw_score unique: ['是' '否'], accuracy=0.67\n",
      "evaluating for type_standard_deviation ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a691261bc1cb4516945e3b25dcd1270c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key=type_standard_deviation raw_score unique: ['是' '否'], accuracy=0.52\n",
      "evaluating for type_interquartile ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "296ee76ebd6c4297876257e6b7be2c88",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key=type_interquartile raw_score unique: ['否' '是'], accuracy=0.64\n",
      "evaluating for type_gradient ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f984320b8d64e45965ee5bb1bdd987f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key=type_gradient raw_score unique: ['是' '否'\n",
      " '否\\n\\n理由：虽然助手的回答提供了更多的细节和具体例子，但是超出了参考答案的范围，包括了新能源发展的详细信息，而参考答案只提到了绿色经济转型和经济多元化改革。根据题目要求，评价应仅基于是否与参考答案相符，因此回答为“否”。但需要说明的是，助手的回答在内容上是正确的，只是超出了题目要求的范围。由于题目要求仅回复“是”或“否”，故不包含此解释部分。'], accuracy=0.66\n",
      "evaluating for clustering ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7775735a6c3435cbf188119256fabc5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key=clustering raw_score unique: ['否' '是'], accuracy=0.74\n",
      "evaluating for type_percentile_further_split ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31193e5295ee420bb7c2b1420247914e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key=type_percentile_further_split raw_score unique: ['是' '否'\n",
      " '否\\n\\n原因是助手的回答虽然包含了参考答案中的国家，但是范围更广，提到了海湾六国和卡塔尔，而不仅仅是沙特阿拉伯、阿联酋和科威特。同时，助手的回答还提供了一些额外的信息，解释了这些国家为何需要发展可再生能源。然而，根据题目要求，只需要判断助手的回答是否与参考答案完全一致，因此在此情况下，答案为“否”。'\n",
      " '否\\n\\n理由：虽然助手的回答提供了更详细的解释，但并没有直接提及参考答案中的“增加财政体系的脆弱性”。然而，助手的回答中提到的债券违约风险上升、融资成本提高、债券市场融资功能减弱以及国债收益率大幅波动等后果，确实可以理解为增加了财政体系的脆弱性。因此，从这个角度来看，助手的回答实际上与参考答案相符，但由于题目要求仅回复“是”或“否”，所以按照字面意思判断，答案应为“否”。但在实际情境下，助手的回答是正确的。'], accuracy=0.67\n"
     ]
    }
   ],
   "source": [
    "metrics = []\n",
    "\n",
    "for key, vector_db in vector_db_dict.items():\n",
    "    print(f'evaluating for {key} ...')\n",
    "    if key in eval_dict:\n",
    "        pred_df = eval_dict[key]\n",
    "    else:\n",
    "        pred_df = pred_dict[key]\n",
    "        pred_df['raw_score'] = evaluate(pred_df)\n",
    "        eval_dict[key] = pred_df\n",
    "        pred_df['score'] = (pred_df['raw_score'] == '是').astype(int)\n",
    "        \n",
    "    print(f\"key={key} raw_score unique: {pred_df['raw_score'].unique()}, accuracy={pred_df['score'].mean()}\")\n",
    "    metrics.append({\n",
    "        'key': key,\n",
    "        'accuracy': pred_df['score'].mean()\n",
    "    })\n",
    "    pickle.dump(eval_dict, open(eval_save_path, 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "094615dd-3ce3-4b2b-a9b2-0f0044394891",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T10:17:53.350533Z",
     "iopub.status.busy": "2024-12-14T10:17:53.349818Z",
     "iopub.status.idle": "2024-12-14T10:17:53.361178Z",
     "shell.execute_reply": "2024-12-14T10:17:53.358917Z",
     "shell.execute_reply.started": "2024-12-14T10:17:53.350464Z"
    }
   },
   "outputs": [],
   "source": [
    "metrics_df = pd.DataFrame(metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "c7702b66-c5da-4a59-95a1-83a377a3323a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T10:17:53.364665Z",
     "iopub.status.busy": "2024-12-14T10:17:53.363927Z",
     "iopub.status.idle": "2024-12-14T10:17:53.393944Z",
     "shell.execute_reply": "2024-12-14T10:17:53.391572Z",
     "shell.execute_reply.started": "2024-12-14T10:17:53.364595Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>type_percentile</td>\n",
       "      <td>0.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>type_standard_deviation</td>\n",
       "      <td>0.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>type_interquartile</td>\n",
       "      <td>0.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>type_gradient</td>\n",
       "      <td>0.66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>clustering</td>\n",
       "      <td>0.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>type_percentile_further_split</td>\n",
       "      <td>0.67</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             key  accuracy\n",
       "0                type_percentile      0.67\n",
       "1        type_standard_deviation      0.52\n",
       "2             type_interquartile      0.64\n",
       "3                  type_gradient      0.66\n",
       "4                     clustering      0.74\n",
       "5  type_percentile_further_split      0.67"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "b537c8af-bc72-40be-ba57-a613ebf531f1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-14T10:24:27.747333Z",
     "iopub.status.busy": "2024-12-14T10:24:27.746502Z",
     "iopub.status.idle": "2024-12-14T10:24:27.886741Z",
     "shell.execute_reply": "2024-12-14T10:24:27.886306Z",
     "shell.execute_reply.started": "2024-12-14T10:24:27.747263Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='accuracy', ylabel='key'>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2QAAAFzCAYAAABRpMrCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAABP60lEQVR4nO3dfXyO9f//8edps+urpmHTGEZG5jIioSSiIn1ZCNMon/JRWORTs83VJIryoT5ko9ikovoqyWrMclFqSyxsWZtaiWLGxzbb+fuj386v02ZXxjH2uN9u5+224zjex3G8jvdO7On9Pt8zmc1mswAAAAAA11wdowsAAAAAgNqKQAYAAAAABiGQAQAAAIBBCGQAAAAAYBACGQAAAAAYhEAGAAAAAAYhkAEAAACAQQhkAAAAAGAQW6MLAFA5RUVF+vXXX+Xq6iqTyWR0OQAAALiE2WzWmTNn5OPjozp1yh4DI5AB15lff/1Vvr6+RpcBAACAcmRlZemWW24psw2BDLjOuLq6Svr7D7ibm5vB1QAAAOBSOTk58vX1tfzcVhYCGXCdKZ6m6ObmRiADAACowSry8RIW9QAAAAAAgxDIAAAAAMAgTFkErlM9X4yVjb2j0WUAAIAbwL6XRxtdQq3FCBkAAAAAGIRABgAAAAAGIZABAAAAgEEIZAAAAABgEAIZAAAAABiEQAYAAAAABiGQAQAAAIBBCGQAAAAAYBACGQAAAAAYhEAGAAAAAAYhkAEAAACAQQhkAAAAAGAQAhkAAAAAGIRABgAAAAAGIZAB1SgmJkYeHh6W7YiICLVv396wegAAAFCzEchqid69e+vZZ581uowbip+fnxYvXmy1LygoSIcPHzamIAAAAFx3bI0uAKhOZrNZhYWFsrU15q3t6OgoR0dHQ+4NAACA6w8jZLVAcHCwtm/friVLlshkMslkMsnW1lYLFy60apecnCyTyaS0tDRJkslk0vLly3X//ffL0dFRzZo103vvvWd1TlZWloYNGyYPDw95enpq0KBBysjIqHBdgwcPVmRkpLy8vOTm5qYJEyYoPz/f0qaoqEhRUVFq2rSpHB0d1a5dO6saEhISZDKZ9Omnn6pTp06yt7fXzp07VVRUpAULFsjf31/29vZq3Lix5s6dW+G6i2tbuHChvL29Va9ePT399NMqKCiQ9PeI488//6zJkydb+lQqOWWxNCtXrlRAQIAcHBzUqlUrLVu2rEL9BQAAgBsPgawWWLJkibp166bx48crOztb2dnZioyMVHR0tFW76Oho9ezZU/7+/pZ9YWFheuSRR5SSkqKRI0fq0UcfVWpqqiSpoKBA/fr1k6urqxITE5WUlCQXFxf179/fKlSVJT4+XqmpqUpISFBsbKw++OADRUZGWo5HRUVpzZo1euONN3TgwAFNnjxZjz32mLZv3251neeff17z589XamqqAgMDNWPGDM2fP19hYWE6ePCg1q1bpwYNGlSq7i+//FLp6en68ssvtXr1asXExCgmJkaS9MEHH+iWW27RrFmzLH1aEWvXrtXMmTM1d+5cpaamat68eQoLC9Pq1asve05eXp5ycnKsXgAAALgxMGWxFnB3d5ednZ2cnJzUsGFDSdLYsWMVHh6uvXv3qkuXLiooKNC6detKjJoNHTpU48aNkyTNnj1bn3/+uV5//XUtW7ZM69evV1FRkVauXGkZIYqOjpaHh4cSEhJ03333lVubnZ2dVq1aJScnJ7Vp00azZs3Sc889p9mzZ6ugoEDz5s3Ttm3b1K1bN0lSs2bNtHPnTr355pvq1auX5TqzZs1S3759JUlnzpzRkiVLtHTpUo0ZM0aS1Lx5c/Xo0UOSKlz3TTfdpKVLl8rGxkatWrXSwIEDFR8fr/Hjx8vT01M2NjZydXW19GlFhIeHa9GiRRoyZIgkqWnTpjp48KDefPNNS62XioqKsgqpAAAAuHEQyGopHx8fDRw4UKtWrVKXLl308ccfKy8vT0OHDrVqVxyELt5OTk6WJKWkpCgtLU2urq5Wbc6fP6/09PQK1dGuXTs5OTlZXT83N1dZWVnKzc3VuXPnLEGrWH5+vjp06GC1r3PnzpavU1NTlZeXpz59+pR6z4rW3aZNG9nY2Fi2vb29tX///go9V2nOnj2r9PR0hYSEaPz48Zb9Fy5ckLu7+2XPmzFjhqZMmWLZzsnJka+vb5XrAAAAQM1BIKvFxo0bp1GjRunVV19VdHS0goKCrMJReXJzc9WpUyetXbu2xDEvL68rri83N1eStHnzZjVq1MjqmL29vdW2s7Oz5evyFtWoaN1169a1OmYymVRUVFSx4i9zX0lasWKFunbtanXs4uB3KXt7+xLPCwAAgBsDgayWsLOzU2FhodW+AQMGyNnZWcuXL9eWLVu0Y8eOEuft3r1bo0ePttouHp3q2LGj1q9fr/r168vNza1KdaWkpOi///2vJUTt3r1bLi4u8vX1laenp+zt7ZWZmWk1PbE8LVq0kKOjo+Lj4y3TLS9WHXVLpfdpWRo0aCAfHx/99NNPGjlyZJXvCwAAgBsHi3rUEn5+ftqzZ48yMjJ04sQJFRUVycbGRsHBwZoxY4ZatGhRYnqiJG3YsEGrVq3S4cOHLZ85mzhxoiRp5MiRuvnmmzVo0CAlJibq6NGjSkhI0KRJk3Ts2LEK1ZWfn6+QkBAdPHhQn3zyicLDwzVx4kTVqVNHrq6uCg0N1eTJk7V69Wqlp6fr22+/1euvv17mIhgODg6aPn26pk2bpjVr1ig9PV27d+/WW2+9VW11F/fpjh079Msvv+jEiRMVOicyMlJRUVF67bXXdPjwYe3fv1/R0dF65ZVXKnxfAAAA3DgIZLVEaGiobGxs1Lp1a3l5eSkzM1OSFBISovz8fI0dO7bU8yIjIxUXF6fAwECtWbNGsbGxat26tSTJyclJO3bsUOPGjTVkyBAFBAQoJCRE58+fr/DIU58+fdSiRQv17NlTQUFBeuihhxQREWE5Pnv2bIWFhSkqKkoBAQHq37+/Nm/erKZNm5Z53bCwME2dOlUzZ85UQECAgoKCdPz48WqrW/p7IZGMjAw1b968wlM0x40bp5UrVyo6Olpt27ZVr169FBMTU+7zAAAA4MZkMpvNZqOLgHESExPVp08fZWVlWZaFL2YymbRx40YNHjz4qtw7ODhYp06d0qZNm67K9W9UOTk5cnd3V7t/viEbe34JNQAAuHL7Xh5dfiNUWPHPa6dPny73P/z5DFktlZeXpz/++EMREREaOnRoiTAGAAAA4OpjymItFRsbqyZNmujUqVNasGDBVbmHi4vLZV+JiYlX5Z4AAADA9YQRsloqODhYwcHBZba50tmsxb+vrDSNGjXSXXfddUXXBwAAAK53BDJcNf7+/kaXAAAAANRoTFkEAAAAAIMQyAAAAADAIAQyAAAAADAIgQwAAAAADEIgAwAAAACDEMgAAAAAwCAEMgAAAAAwCIEMAAAAAAxCIAMAAAAAgxDIAAAAAMAgtkYXAKBqdswZLjc3N6PLAAAAwBVghAwAAAAADEIgAwAAAACDEMgAAAAAwCAEMgAAAAAwCIEMAAAAAAxCIAMAAAAAgxDIAAAAAMAgBDIAAAAAMAiBDAAAAAAMYmt0AQCqJmv+HXJ1sDG6DABALdF45n6jSwBuSIyQAQAAAIBBCGQAAAAAYBACGQAAAAAYhEAGAAAAAAYhkAEAAACAQQhkAAAAAGAQAhkAAAAAGIRABgAAAAAGIZABAAAAgEEIZAAAAABgEAIZAAAAABiEQAYAAAAABiGQAQAAAIBBCGQAAAAAYBACGQAAAAAYhEAGSVJwcLAGDx5cI6/du3dvPfvss9VWT0ZGhkwmk5KTk2vEdQAAAFB71ZpAVt0/1F9tERERat++vdFl3JB8fX2VnZ2t2267rcLnlBYqq3IdAAAA4GK2RheAG0dBQYHq1q1rdBnlsrGxUcOGDWvMdQAAAFB71YoRsuDgYG3fvl1LliyRyWSSyWSSra2tFi5caNUuOTlZJpNJaWlpkiSTyaTly5fr/vvvl6Ojo5o1a6b33nvP6pysrCwNGzZMHh4e8vT01KBBg5SRkVGhuhISEtSlSxc5OzvLw8NDd955p37++WfFxMQoMjJSKSkplnpjYmIkSa+88oratm0rZ2dn+fr66qmnnlJubq7lmjExMfLw8NBnn32mgIAAubi4qH///srOzra0KSws1JQpU+Th4aF69epp2rRpMpvNVrVt2bJFPXr0sLR54IEHlJ6ebjlePF1v/fr16tWrlxwcHLR27doKXbssZ8+e1ejRo+Xi4iJvb28tWrSoRJu8vDyFhoaqUaNGcnZ2VteuXZWQkCBJysnJkaOjoz799FOrczZu3ChXV1edO3euxFTDwsJChYSEqGnTpnJ0dNStt96qJUuWWM6NiIjQ6tWr9eGHH1q+HwkJCaVOWdy+fbu6dOkie3t7eXt76/nnn9eFCxcsx3v37q1JkyZp2rRp8vT0VMOGDRUREVHh/gEAAMCNpVYEsiVLlqhbt24aP368srOzlZ2drcjISEVHR1u1i46OVs+ePeXv72/ZFxYWpkceeUQpKSkaOXKkHn30UaWmpkr6e0SoX79+cnV1VWJiopKSkiwBKD8/v8yaLly4oMGDB6tXr176/vvvtWvXLj3xxBMymUwKCgrS1KlT1aZNG0u9QUFBkqQ6derotdde04EDB7R69Wp98cUXmjZtmtW1z507p4ULF+rtt9/Wjh07lJmZqdDQUMvxRYsWKSYmRqtWrdLOnTv1559/auPGjVbXOHv2rKZMmaJvvvlG8fHxqlOnjh5++GEVFRVZtXv++ef1zDPPKDU1Vf369avQtcvy3HPPafv27frwww+1detWJSQk6Ntvv7VqM3HiRO3atUtxcXH6/vvvNXToUPXv319HjhyRm5ubHnjgAa1bt87qnLVr12rw4MFycnIqcc+ioiLdcsst2rBhgw4ePKiZM2fqX//6l959911JUmhoqIYNG2YJttnZ2erevXuJ6/zyyy8aMGCAbr/9dqWkpGj58uV66623NGfOHKt2q1evlrOzs/bs2aMFCxZo1qxZ+vzzzy/bJ3l5ecrJybF6AQAA4MZgMldm+OI61rt3b7Vv316LFy+WJP36669q3LixvvrqK3Xp0kUFBQXy8fHRwoULNWbMGEl/j5BNmDBBy5cvt1znjjvuUMeOHbVs2TK98847mjNnjlJTU2UymSRJ+fn58vDw0KZNm3Tfffddtp4///xT9erVU0JCgnr16lXieEREhDZt2lTughHvvfeeJkyYoBMnTkj6e4Rs7NixSktLU/PmzSVJy5Yt06xZs/Tbb79Jknx8fDR58mQ999xzkv4Oh02bNlWnTp20adOmUu9z4sQJeXl5af/+/brtttuUkZGhpk2bavHixXrmmWcs7apy7WK5ubmqV6+e3nnnHQ0dOtTST7fccoueeOIJLV68WJmZmWrWrJkyMzPl4+NjOffee+9Vly5dNG/ePG3atEmjRo3S77//LicnJ+Xk5KhBgwbauHGj+vfvb6n9u+++u+zn9CZOnKjffvvNMiIaHBysU6dOWT3Dpdd54YUX9P7771u9H5YtW6bp06fr9OnTqlOnjnr37q3CwkIlJiZartOlSxfdc889mj9/fqm1REREKDIyssT+H2YEyNXBpsw+BQCgujSeud/oEoDrRk5Ojtzd3XX69Gm5ubmV2bZWjJCVxsfHRwMHDtSqVaskSR9//LHy8vIsQaBYt27dSmwXj5ClpKQoLS1Nrq6ucnFxkYuLizw9PXX+/Hmr6X2l8fT0VHBwsPr166cHH3xQS5YssZpWeDnbtm1Tnz591KhRI7m6umrUqFE6efKkzp07Z2nj5ORkCWOS5O3trePHj0uSTp8+rezsbHXt2tVy3NbWVp07d7a6z5EjRzR8+HA1a9ZMbm5u8vPzkyRlZmZatbv4vIpe+3LS09OVn59vdb6np6duvfVWy/b+/ftVWFioli1bWvrcxcVF27dvt/T5gAEDVLduXX300UeSpPfff19ubm669957L3vvf//73+rUqZO8vLzk4uKi//znPyWetTypqanq1q2bJYxJ0p133qnc3FwdO3bMsi8wMNDqvIu/P6WZMWOGTp8+bXllZWVVqi4AAADUXLU2kEnSuHHjFBcXp//+97+Kjo5WUFBQqVPaLic3N1edOnVScnKy1evw4cMaMWJEuedHR0dr165d6t69u9avX6+WLVtq9+7dl22fkZGhBx54QIGBgXr//fe1b98+/fvf/5YkqymSly6sYTKZKvU5Lkl68MEH9eeff2rFihXas2eP9uzZU+I+kuTs7Fyp616p3Nxc2djYaN++fVZ9npqaavncl52dnf7nf/7HMm1x3bp1CgoKkq1t6WvYxMXFKTQ0VCEhIdq6dauSk5M1duzYcqedVlVp359Lp4JezN7eXm5ublYvAAAA3BhqTSCzs7NTYWGh1b4BAwbI2dlZy5cv15YtW/T444+XOO/SgLR7924FBARIkjp27KgjR46ofv368vf3t3q5u7tXqK4OHTpoxowZ+uqrr3TbbbdZQkRp9e7bt09FRUVatGiR7rjjDrVs2VK//vprhftAktzd3eXt7W0JWNLf0wr37dtn2T558qQOHTqkF198UX369FFAQID++uuvarl2WZo3b666detanf/XX3/p8OHDlu0OHTqosLBQx48fL9HnF694OHLkSG3ZskUHDhzQF198oZEjR172vklJSerevbueeuopdejQQf7+/iVGOEv7flwqICBAu3btsgq/SUlJcnV11S233FKhPgAAAEDtUmsCmZ+fn/bs2aOMjAydOHFCRUVFsrGxUXBwsGbMmKEWLVqUmJ4oSRs2bNCqVat0+PBhhYeHa+/evZo4caKkv3/ov/nmmzVo0CAlJibq6NGjSkhI0KRJk6ymqJXm6NGjmjFjhnbt2qWff/5ZW7du1ZEjRyxhz8/PT0ePHlVycrJOnDihvLw8+fv7q6CgQK+//rp++uknvf3223rjjTcq3RfPPPOM5s+fr02bNunHH3/UU089pVOnTlmO33TTTapXr57+85//KC0tTV988YWmTJlSLdcui4uLi0JCQvTcc8/piy++0A8//KDg4GDVqfN/b9OWLVtq5MiRGj16tD744AMdPXpUe/fuVVRUlDZv3mxp17NnTzVs2FAjR45U06ZNraZBXqpFixb65ptv9Nlnn+nw4cMKCwvT119/bdXGz89P33//vQ4dOqQTJ06ooKCgxHWeeuopZWVl6Z///Kd+/PFHffjhhwoPD9eUKVOsngEAAAAoVmt+SgwNDZWNjY1at24tLy8vy+eDQkJClJ+fr7Fjx5Z6XmRkpOLi4hQYGKg1a9YoNjZWrVu3lvT3Z7V27Nihxo0ba8iQIQoICFBISIjOnz9f7rQyJycn/fjjj3rkkUfUsmVLPfHEE3r66af15JNPSpIeeeQR9e/fX3fffbe8vLwUGxurdu3a6ZVXXtFLL72k2267TWvXrlVUVFSl+2Lq1KkaNWqUxowZo27dusnV1VUPP/yw5XidOnUUFxenffv26bbbbtPkyZP18ssvV8u1y/Pyyy/rrrvu0oMPPqh7771XPXr0UKdOnazaREdHa/To0Zo6dapuvfVWDR48WF9//bUaN25saWMymTR8+HDL6phlefLJJzVkyBAFBQWpa9euOnnypJ566imrNuPHj9ett96qzp07y8vLS0lJSSWu06hRI33yySfau3ev2rVrpwkTJigkJEQvvvhihZ8fAAAAtUutWWXxchITE9WnTx9lZWWpQYMGVsdMJpM2btyowYMHG1McUIriVXtYZREAcC2xyiJQcZVZZbH0VQ5qgby8PP3xxx+KiIjQ0KFDS4QxAAAAALjaas2UxUvFxsaqSZMmOnXqlBYsWHBV7nHxsuyXvi7+PVS1RWZmZpl9Utll5gEAAIDrXa0dIQsODlZwcHCZba50NmdZv9S5UaNGV3Tt65GPj0+ZfXLxL3oGAAAAaoNaG8iuBX9/f6NLqFFsbW3pEwAAAOAitXbKIgAAAAAYjUAGAAAAAAYhkAEAAACAQQhkAAAAAGAQAhkAAAAAGIRABgAAAAAGIZABAAAAgEEIZAAAAABgEAIZAAAAABjE1ugCAFSN7/O75ebmZnQZAAAAuAKMkAEAAACAQQhkAAAAAGAQAhkAAAAAGIRABgAAAAAGIZABAAAAgEEIZAAAAABgEAIZAAAAABiEQAYAAAAABiGQAQAAAIBBCGQAAAAAYBBbowsAUDV93+grW0f+CAMAUBsk/TPJ6BJwlTBCBgAAAAAGIZABAAAAgEEIZAAAAABgEAIZAAAAABiEQAYAAAAABiGQAQAAAIBBCGQAAAAAYBACGQAAAAAYhEAGAAAAAAYhkAEAAACAQQhkAAAAAGAQAhkAAAAAGIRABgAAAAAGIZABAAAAgEEIZKiy4OBgDR482OgyahyTyaRNmzZJkjIyMmQymZScnGxoTQAAAKiZCGTXid69e+vZZ581ugwrS5YsUUxMTKXOuTisXO8iIiLUvn37Evuzs7N1//33X/uCAAAAcN2xNboAXL/c3d0Nu3dBQYHq1q1ryL3NZrMKCwsve7xhw4bXsBoAAABczxghuw4EBwdr+/btWrJkiUwmk0wmk2xtbbVw4UKrdsnJyTKZTEpLS5P092jU8uXLdf/998vR0VHNmjXTe++9Z3VOVlaWhg0bJg8PD3l6emrQoEHKyMiocF0XT1ns3bu3Jk2apGnTpsnT01MNGzZURESE5bifn58k6eGHH5bJZLJsS9KHH36ojh07ysHBQc2aNVNkZKQuXLhgOV78LA899JCcnZ01d+5cSdL8+fPVoEEDubq6KiQkRM8//7zVqFVpI4uDBw9WcHCwZfvtt99W586d5erqqoYNG2rEiBE6fvy45XhCQoJMJpM+/fRTderUSfb29nrnnXcUGRmplJQUy/ekeLSwvFHAH374Qffff79cXFzUoEEDjRo1SidOnCizrwEAAHBjIpBdB5YsWaJu3bpp/Pjxys7OVnZ2tiIjIxUdHW3VLjo6Wj179pS/v79lX1hYmB555BGlpKRo5MiRevTRR5Wamirp71Gmfv36ydXVVYmJiUpKSpKLi4v69++v/Pz8KtW6evVqOTs7a8+ePVqwYIFmzZqlzz//XJL09ddfW+rMzs62bCcmJmr06NF65plndPDgQb355puKiYmxhK5iERERevjhh7V//349/vjjevfddxUREaF58+bpm2++kbe3t5YtW1bpmgsKCjR79mylpKRo06ZNysjIsApsxZ5//nnNnz9fqamp6tu3r6ZOnao2bdpYvidBQUHl3uvUqVO655571KFDB33zzTfasmWLfv/9dw0bNuyy5+Tl5SknJ8fqBQAAgBsDUxavA+7u7rKzs5OTk5NlOtzYsWMVHh6uvXv3qkuXLiooKNC6detKjJoNHTpU48aNkyTNnj1bn3/+uV5//XUtW7ZM69evV1FRkVauXCmTySTp77Dk4eGhhIQE3XfffZWuNTAwUOHh4ZKkFi1aaOnSpYqPj1ffvn3l5eUlSfLw8LCa1hcZGannn39eY8aMkSQ1a9ZMs2fP1rRp0yzXkqQRI0Zo7Nixlu1HH31UISEhCgkJkSTNmTNH27Zt0/nz5ytV8+OPP275ulmzZnrttdd0++23Kzc3Vy4uLpZjs2bNUt++fS3bLi4usrW1rdQUxaVLl6pDhw6aN2+eZd+qVavk6+urw4cPq2XLliXOiYqKUmRkZKWeCQAAANcHRsiuUz4+Pho4cKBWrVolSfr444+Vl5enoUOHWrXr1q1bie3iEbKUlBSlpaXJ1dVVLi4ucnFxkaenp86fP6/09PQq1RUYGGi17e3tbTX9rzQpKSmaNWuWpQYXFxfLaOC5c+cs7Tp37mx1Xmpqqrp27Vri+Spr3759evDBB9W4cWO5urqqV69ekqTMzEyrdpfevypSUlL05ZdfWj1rq1atJOmyfT5jxgydPn3a8srKyrriOgAAAFAzMEJ2HRs3bpxGjRqlV199VdHR0QoKCpKTk1OFz8/NzVWnTp20du3aEseKR7Mq69KFNkwmk4qKisqtIzIyUkOGDClxzMHBwfK1s7NzpeupU6eOzGaz1b6CggLL12fPnlW/fv3Ur18/rV27Vl5eXsrMzFS/fv1KTNusyv0vlZubqwcffFAvvfRSiWPe3t6lnmNvby97e/srvjcAAABqHgLZdcLOzq7Eyn4DBgyQs7Ozli9fri1btmjHjh0lztu9e7dGjx5ttd2hQwdJUseOHbV+/XrVr19fbm5uV/cB/r+6deuWeI6OHTvq0KFDVp99q4iAgADt2bOnxPNdzMvLS9nZ2ZbtwsJC/fDDD7r77rslST/++KNOnjyp+fPny9fXV5L0zTffVOj+pX1PytOxY0e9//778vPzk60tf/wAAABqO6YsXif8/Py0Z88eZWRk6MSJEyoqKpKNjY2Cg4M1Y8YMtWjRotTpehs2bNCqVat0+PBhy2fOJk6cKEkaOXKkbr75Zg0aNEiJiYk6evSoEhISNGnSJB07duyqPUd8fLx+++03/fXXX5KkmTNnas2aNYqMjNSBAweUmpqquLg4vfjii2Ve65lnntGqVasUHR1teb4DBw5Ytbnnnnu0efNmbd68WT/++KP+8Y9/6NSpU5bjjRs3lp2dnV5//XX99NNP+uijjzR79uwKP8vRo0eVnJysEydOKC8vr9xznn76af35558aPny4vv76a6Wnp+uzzz7T2LFjKx3uAAAAcP0jkF0nQkNDZWNjo9atW1um1UlSSEiI8vPzrRa7uFhkZKTi4uIUGBioNWvWKDY2Vq1bt5YkOTk5aceOHWrcuLGGDBmigIAAhYSE6Pz581dtxGzRokX6/PPP5evraxmp69evn/73f/9XW7du1e2336477rhDr776qpo0aVLmtYKCghQWFqZp06apU6dO+vnnn/WPf/zDqs3jjz+uMWPGaPTo0erVq5eaNWtmGR2T/h5Bi4mJ0YYNG9S6dWvNnz+/xMIol/PII4+of//+uvvuu+Xl5aXY2Nhyz/Hx8VFSUpIKCwt13333qW3btnr22Wfl4eGhOnX44wgAAFDbmMyXfsAG15XExET16dNHWVlZatCggdUxk8mkjRs3Wv2usBtdRESENm3apOTkZKNLuWpycnLk7u6uLi91ka0j0x4BAKgNkv6ZZHQJqITin9dOnz5d7kAHP81dp/Ly8vTHH38oIiJCQ4cOLRHGAAAAANR8zJG6TsXGxqpJkyY6deqUFixYcFXucfHS7Je+EhMTr8o9AQAAgNqEKYu4rLS0tMsea9SokRwdHa9hNSjGlEUAAGofpixeX5iyiGpR2WXoAQAAAFQOUxYBAAAAwCAEMgAAAAAwCIEMAAAAAAxCIAMAAAAAgxDIAAAAAMAgBDIAAAAAMEiVAlmvXr20Zs0a/fe//63uegAAAACg1qhSIOvQoYNCQ0PVsGFDjR8/Xrt3767uugAAAADghlelQLZ48WL9+uuvio6O1vHjx9WzZ0+1bt1aCxcu1O+//17dNQIAAADADanKnyGztbXVkCFD9OGHH+rYsWMaMWKEwsLC5Ovrq8GDB+uLL76ozjoBAAAA4IZje6UX2Lt3r6KjoxUXF6f69esrODhYv/zyix544AE99dRTWrhwYXXUCeASn0/4XG5ubkaXAQAAgCtgMpvN5sqedPz4cb399tuKjo7WkSNH9OCDD2rcuHHq16+fTCaTJGnnzp3q37+/cnNzq71ooDbLycmRu7u7Tp8+TSADAACogSrz81qVRshuueUWNW/eXI8//riCg4Pl5eVVok1gYKBuv/32qlweAAAAAGqFKgWy+Ph43XXXXWW2cXNz05dfflmlogAAAACgNqjSoh533XWXLly4oG3btunNN9/UmTNnJEm//vorUxQBAAAAoIKqNEL2888/q3///srMzFReXp769u0rV1dXvfTSS8rLy9Mbb7xR3XUCAAAAwA2nSiNkzzzzjDp37qy//vpLjo6Olv0PP/yw4uPjq604AAAAALiRVWmELDExUV999ZXs7Oys9vv5+emXX36plsIAAAAA4EZXpRGyoqIiFRYWlth/7Ngxubq6XnFRAAAAAFAbVCmQ9e3bV4sXL7Zsm0wm5ebmKjw8XAMGDKiu2gAAAADghlalXwx97Ngx9evXT2azWUeOHFHnzp115MgR1atXT4mJiapfv/7VqBWA/u8XDW7u1l3OtlWadQwAAGqZXju2G11CrVKZXwxdpRGyxMREpaSk6F//+pcmT56sDh06aP78+UpOTtbLL79cpaIBAAAAoLap0n+v/+Mf/5CHh4cee+wxq/1TpkxRbGwsoQwAAAAAKqBKI2Rr167V8OHDtXPnTsu+SZMmKS4uTl9++WW1FQcAAAAAN7IqBbKBAwdq2bJleuihh7Rv3z499dRTev/995WQkKBWrVpVd40AAAAAcEOq8ooAI0aM0KlTp3TnnXfKy8tL27dvl7+/f3XWBgAAAAA3tAoHsilTppS638vLSx07dtSyZcss+1555ZUrrwwAAAAAbnAVDmTfffddqfv9/f2Vk5NjOW4ymaqnMgAAAAC4wVU4kLFYBwAAAABUryot6gEAAAAAuHIEMgAAAAAwCIEMAAAAAAxCIAMAAAAAgxDIAAAAAMAgBDIAAAAAMAiBDChFTEyMPDw8LNsRERFq3769YfUAAADgxkQgQwm9e/fWs88+a3QZNUpoaKji4+Or9ZqXhj4AAADUPhX+xdDA9SY/P192dnbVci0XFxe5uLhUy7UAAACAYoyQwUpwcLC2b9+uJUuWyGQyyWQyydbWVgsXLrRql5ycLJPJpLS0NEmSyWTS8uXLdf/998vR0VHNmjXTe++9Z3VOVlaWhg0bJg8PD3l6emrQoEHKyMioUF0XLlzQpEmT5OHhoXr16mn69OkaM2aMBg8ebGnTu3dvTZw4Uc8++6xuvvlm9evXT5L0yiuvqG3btnJ2dpavr6+eeuop5ebmWl0/JiZGjRs3lpOTkx5++GGdPHnS6nhpUxZXrlypgIAAOTg4qFWrVlq2bJnlWEZGhkwmkz744APdfffdcnJyUrt27bRr1y5JUkJCgsaOHavTp09b+jkiIqJCfQEAAIAbB4EMVpYsWaJu3bpp/Pjxys7OVnZ2tiIjIxUdHW3VLjo6Wj179pS/v79lX1hYmB555BGlpKRo5MiRevTRR5WamipJKigoUL9+/eTq6qrExEQlJSXJxcVF/fv3V35+frl1vfTSS1q7dq2io6OVlJSknJwcbdq0qUS71atXy87OTklJSXrjjTckSXXq1NFrr72mAwcOaPXq1friiy80bdo0yzl79uxRSEiIJk6cqOTkZN19992aM2dOmfWsXbtWM2fO1Ny5c5Wamqp58+YpLCxMq1evtmr3wgsvKDQ0VMnJyWrZsqWGDx+uCxcuqHv37lq8eLHc3Nws/RwaGlrqvfLy8pSTk2P1AgAAwI3BZDabzUYXgZqld+/eat++vRYvXixJ+vXXX9W4cWN99dVX6tKliwoKCuTj46OFCxdqzJgxkv4eIZswYYKWL19uuc4dd9yhjh07atmyZXrnnXc0Z84cpaamymQySfp7SqGHh4c2bdqk++67r8yaGjZsqNDQUEtoKSwsVLNmzdShQwdLMOvdu7dycnL07bfflnmt9957TxMmTNCJEyckSSNGjNDp06e1efNmS5tHH31UW7Zs0alTpyT9PUK2adMmJScnS5L8/f01e/ZsDR8+3HLOnDlz9Mknn+irr75SRkaGmjZtqpUrVyokJESSdPDgQbVp00apqalq1aqVYmJi9Oyzz1rucTkRERGKjIwssX9zt+5ytmXWMQAAKF+vHduNLqFWycnJkbu7u06fPi03N7cy2zJChnL5+Pho4MCBWrVqlSTp448/Vl5enoYOHWrVrlu3biW2i0fIUlJSlJaWJldXV8vnsTw9PXX+/Hmlp6eXef/Tp0/r999/V5cuXSz7bGxs1KlTpxJtS9u3bds29enTR40aNZKrq6tGjRqlkydP6ty5c5Kk1NRUde3atcxnudjZs2eVnp6ukJAQy7O4uLhozpw5JZ4lMDDQ8rW3t7ck6fjx42U+76VmzJih06dPW15ZWVmVOh8AAAA1F/+9jgoZN26cRo0apVdffVXR0dEKCgqSk5NThc/Pzc1Vp06dtHbt2hLHvLy8qq1OZ2dnq+2MjAw98MAD+sc//qG5c+fK09NTO3fuVEhIiPLz8yv1DMWKP3+2YsWKEkHOxsbGartu3bqWr4tHBouKiip1P3t7e9nb21e6TgAAANR8BDKUYGdnp8LCQqt9AwYMkLOzs5YvX64tW7Zox44dJc7bvXu3Ro8ebbXdoUMHSVLHjh21fv161a9fv9xh20u5u7urQYMG+vrrr9WzZ09Jf09Z/Pbbb8v93WD79u1TUVGRFi1apDp1/h4Qfvfdd63aBAQEaM+ePSWe5XIaNGggHx8f/fTTTxo5cmSlnuVipfUzAAAAahemLKIEPz8/7dmzRxkZGTpx4oSKiopkY2Oj4OBgzZgxQy1atCh1St+GDRu0atUqHT58WOHh4dq7d68mTpwoSRo5cqRuvvlmDRo0SImJiTp69KgSEhI0adIkHTt2rNya/vnPfyoqKkoffvihDh06pGeeeUZ//fWXZdTpcvz9/VVQUKDXX39dP/30k95++23LYh/FJk2apC1btmjhwoU6cuSIli5dqi1btpR53cjISEVFRem1117T4cOHtX//fkVHR+uVV14p91mK+fn5KTc3V/Hx8Tpx4oRlCiUAAABqDwIZSggNDZWNjY1at24tLy8vZWZmSpJlmt/YsWNLPS8yMlJxcXEKDAzUmjVrFBsbq9atW0uSnJyctGPHDjVu3FhDhgxRQECAQkJCdP78+QqNmE2fPl3Dhw/X6NGj1a1bN7m4uKhfv35ycHAo87x27drplVde0UsvvaTbbrtNa9euVVRUlFWbO+64QytWrNCSJUvUrl07bd26VS+++GKZ1x03bpxWrlyp6OhotW3bVr169VJMTIyaNm1a7rMU6969uyZMmKCgoCB5eXlpwYIFFT4XAAAANwZWWUSFJSYmqk+fPsrKylKDBg2sjplMJm3cuNHq94JdTUVFRQoICNCwYcM0e/bsa3LPmqJ41R5WWQQAABXFKovXVmVWWeSnOZQrLy9Pf/zxhyIiIjR06NASYexa+Pnnn7V161b16tVLeXl5Wrp0qY4ePaoRI0Zc81oAAACA6sKURZQrNjZWTZo00alTp67atLqLl4+/9JWYmKg6deooJiZGt99+u+68807t379f27ZtU0BAwFWpBwAAALgWmLKIGiEtLe2yxxo1aiRHR8drWE3NxpRFAABQWUxZvLaYsojrjr+/v9ElAAAAANccUxYBAAAAwCAEMgAAAAAwCIEMAAAAAAxCIAMAAAAAgxDIAAAAAMAgBDIAAAAAMAiBDAAAAAAMQiADAAAAAIMQyAAAAADAIAQyAAAAADCIrdEFAKiaHls+lZubm9FlAAAA4AowQgYAAAAABiGQAQAAAIBBCGQAAAAAYBACGQAAAAAYhEAGAAAAAAYhkAEAAACAQQhkAAAAAGAQAhkAAAAAGIRABgAAAAAGsTW6AABV8+a/PpWjvZPRZQAAANR4Exc9aHQJl8UIGQAAAAAYhEAGAAAAAAYhkAEAAACAQQhkAAAAAGAQAhkAAAAAGIRABgAAAAAGIZABAAAAgEEIZAAAAABgEAIZAAAAABiEQAYAAAAABiGQAQAAAIBBCGQAAAAAYBACGQAAAAAYhEAGAAAAAAYhkAEAAACAQQhkqNEyMjJkMpmUnJxsdCmlCg4O1uDBg40uAwAAANcpAhlqlZiYGHl4eFTb9ZYsWaKYmJhqux4AAABqF1ujCwCuR4WFhTKZTHJ3dze6FAAAAFzHGCFDjVBUVKQFCxbI399f9vb2aty4sebOnVuiXWkjXJs2bZLJZLJsp6Sk6O6775arq6vc3NzUqVMnffPNN0pISNDYsWN1+vRpmUwmmUwmRURESJLy8vIUGhqqRo0aydnZWV27dlVCQkKJ+3700Udq3bq17O3tlZmZWWLKYu/evTVp0iRNmzZNnp6eatiwoeUexX788Uf16NFDDg4Oat26tbZt2yaTyaRNmzZdYS8CAADgesMIGWqEGTNmaMWKFXr11VfVo0cPZWdn68cff6zStUaOHKkOHTpo+fLlsrGxUXJysurWravu3btr8eLFmjlzpg4dOiRJcnFxkSRNnDhRBw8eVFxcnHx8fLRx40b1799f+/fvV4sWLSRJ586d00svvaSVK1eqXr16ql+/fqn3X716taZMmaI9e/Zo165dCg4O1p133qm+ffuqsLBQgwcPVuPGjbVnzx6dOXNGU6dOLfN58vLylJeXZ9nOycmpUr8AAACg5iGQwXBnzpzRkiVLtHTpUo0ZM0aS1Lx5c/Xo0UMZGRmVvl5mZqaee+45tWrVSpIsgUqS3N3dZTKZ1LBhQ6v20dHRyszMlI+PjyQpNDRUW7ZsUXR0tObNmydJKigo0LJly9SuXbsy7x8YGKjw8HDLvZcuXar4+Hj17dtXn3/+udLT05WQkGCpYe7cuerbt+9lrxcVFaXIyMhK9wMAAABqPqYswnCpqanKy8tTnz59quV6U6ZM0bhx43Tvvfdq/vz5Sk9PL7P9/v37VVhYqJYtW8rFxcXy2r59u9W5dnZ2CgwMLPf+l7bx9vbW8ePHJUmHDh2Sr6+vVSDs0qVLmdebMWOGTp8+bXllZWWVWwMAAACuD4yQwXCOjo4VblunTh2ZzWarfQUFBVbbERERGjFihDZv3qxPP/1U4eHhiouL08MPP1zqNXNzc2VjY6N9+/bJxsbG6ljxlMbiOi/+rNrl1K1b12rbZDKpqKio3PMux97eXvb29lU+HwAAADUXI2QwXIsWLeTo6Kj4+Phy23p5eenMmTM6e/asZV9pv6OsZcuWmjx5srZu3aohQ4YoOjpa0t+jXIWFhVZtO3TooMLCQh0/flz+/v5Wr4tHsqrDrbfeqqysLP3++++WfV9//XW13gMAAADXDwIZDOfg4KDp06dr2rRpWrNmjdLT07V792699dZbJdp27dpVTk5O+te//qX09HStW7fO6veA/fe//9XEiROVkJCgn3/+WUlJSfr6668VEBAgSfLz81Nubq7i4+N14sQJnTt3Ti1bttTIkSM1evRoffDBBzp69Kj27t2rqKgobd68uVqftW/fvmrevLnGjBmj77//XklJSXrxxRclqUKjbwAAALixEMhQI4SFhWnq1KmaOXOmAgICFBQUZPnc1cU8PT31zjvv6JNPPlHbtm0VGxtrtay8jY2NTp48qdGjR6tly5YaNmyY7r//fsuiGN27d9eECRMUFBQkLy8vLViwQJIUHR2t0aNHa+rUqbr11ls1ePBgff3112rcuHG1PqeNjY02bdqk3Nxc3X777Ro3bpxeeOEFSX8HUwAAANQuJvOlH8gBcE0lJSWpR48eSktLU/Pmzcttn5OTI3d3dy14Ok6O9k7XoEIAAIDr28RFD17T+xX/vHb69Gm5ubmV2ZZFPYBrbOPGjXJxcVGLFi2UlpamZ555RnfeeWeFwhgAAABuLAQy4Bo7c+aMpk+frszMTN1888269957tWjRIqPLAgAAgAEIZMA1Nnr0aI0ePdroMgAAAFADsKgHAAAAABiEQAYAAAAABiGQAQAAAIBBCGQAAAAAYBACGQAAAAAYhEAGAAAAAAYhkAEAAACAQQhkAAAAAGAQAhkAAAAAGIRABgAAAAAGsTW6AABV8+S8++Xm5mZ0GQAAALgCjJABAAAAgEEIZAAAAABgEAIZAAAAABiEQAYAAAAABiGQAQAAAIBBCGQAAAAAYBACGQAAAAAYhEAGAAAAAAYhkAEAAACAQQhkAAAAAGAQW6MLAFA1L48fJYe6dY0uAwAA3ABeeOc9o0uotRghAwAAAACDEMgAAAAAwCAEMgAAAAAwCIEMAAAAAAxCIAMAAAAAgxDIAAAAAMAgBDIAAAAAMAiBDAAAAAAMQiADAAAAAIMQyAAAAADAIAQyAAAAADAIgQwAAAAADEIgAwAAAACDEMgAAAAAwCAEslogJiZGHh4elu2IiAi1b9++2q7/448/6o477pCDg0O1XrdYRkaGTCaTkpOTq/3aV9OldSckJMhkMunUqVOG1gUAAICaw9BA1rt3bz377LNGlnDD8fPz0+LFi632BQUF6fDhw1ftnuHh4XJ2dtahQ4cUHx9/RdcKDg7W4MGDq6ewGqZ79+7Kzs6Wu7u7pJJBGQAAALWPrdEF1AZms1mFhYWytTWmux0dHeXo6HjVrp+enq6BAweqSZMmVb5GYWGhTCZTNVZVvvz8fNnZ2V2z+9nZ2alhw4bX7H4AAACo+QwbIQsODtb27du1ZMkSmUwmmUwm2draauHChVbtkpOTZTKZlJaWJkkymUxavny57r//fjk6OqpZs2Z67733rM7JysrSsGHD5OHhIU9PTw0aNEgZGRkVrmvw4MGKjIyUl5eX3NzcNGHCBOXn51vaFBUVKSoqSk2bNpWjo6PatWtnVUPx1LRPP/1UnTp1kr29vXbu3KmioiItWLBA/v7+sre3V+PGjTV37twK111c28KFC+Xt7a169erp6aefVkFBgaS/Rxx//vlnTZ482dKnUsVGYlauXKmAgAA5ODioVatWWrZsWYX6y2Qyad++fZo1a5ZMJpMiIiJKnZpX/H0sfp7imj766CO1bt1a9vb2evzxx7V69Wp9+OGHlvoTEhIs1/jpp5909913y8nJSe3atdOuXbusatm5c6fuuusuOTo6ytfXV5MmTdLZs2ctx/38/DR79myNHj1abm5ueuKJJ8p8tvz8fE2cOFHe3t5ycHBQkyZNFBUVZfXs5b0XL3ZxvyQkJGjs2LE6ffq05VkjIiLK73AAAADcUAwLZEuWLFG3bt00fvx4ZWdnKzs7W5GRkYqOjrZqFx0drZ49e8rf39+yLywsTI888ohSUlI0cuRIPfroo0pNTZUkFRQUqF+/fnJ1dVViYqKSkpLk4uKi/v37W4WqssTHxys1NVUJCQmKjY3VBx98oMjISMvxqKgorVmzRm+88YYOHDigyZMn67HHHtP27dutrvP8889r/vz5Sk1NVWBgoGbMmKH58+crLCxMBw8e1Lp169SgQYNK1f3ll18qPT1dX375pVavXq2YmBjFxMRIkj744APdcsstmjVrlqVPK2Lt2rWaOXOm5s6dq9TUVM2bN09hYWFavXp1uedmZ2erTZs2mjp1qrKzsxUaGlqhe0rSuXPn9NJLL2nlypU6cOCAXnvtNQ0bNkz9+/e31N+9e3dL+xdeeEGhoaFKTk5Wy5YtNXz4cF24cEHS36N0/fv31yOPPKLvv/9e69ev186dOzVx4kSrey5cuFDt2rXTd999p7CwsDLre+211/TRRx/p3Xff1aFDh7R27Vr5+flZtSnrvViW7t27a/HixXJzc7M86+X6Li8vTzk5OVYvAAAA3BgMm7Lo7u4uOzs7OTk5WaZxjR07VuHh4dq7d6+6dOmigoICrVu3rsSo2dChQzVu3DhJ0uzZs/X555/r9ddf17Jly7R+/XoVFRVp5cqVlhGi6OhoeXh4KCEhQffdd1+5tdnZ2WnVqlVycnJSmzZtNGvWLD333HOaPXu2CgoKNG/ePG3btk3dunWTJDVr1kw7d+7Um2++qV69elmuM2vWLPXt21eSdObMGS1ZskRLly7VmDFjJEnNmzdXjx49JKnCdd90001aunSpbGxs1KpVKw0cOFDx8fEaP368PD09ZWNjI1dX10pNjQsPD9eiRYs0ZMgQSVLTpk118OBBvfnmm5ZaL6dhw4aytbWVi4tLpafjFRQUaNmyZWrXrp1ln6Ojo/Ly8kq9VmhoqAYOHChJioyMVJs2bZSWlqZWrVopKipKI0eOtHwmsUWLFnrttdfUq1cvLV++XA4ODpKke+65R1OnTq1QfZmZmWrRooV69Oghk8lU6pTMst6LZbGzs5O7u7tMJlO5/RYVFWX1HwIAAAC4cdSoz5D5+Pho4MCBWrVqlbp06aKPP/5YeXl5Gjp0qFW74iB08XbxSnYpKSlKS0uTq6urVZvz588rPT29QnW0a9dOTk5OVtfPzc1VVlaWcnNzde7cOUvQKpafn68OHTpY7evcubPl69TUVOXl5alPnz6l3rOidbdp00Y2NjaWbW9vb+3fv79Cz1Was2fPKj09XSEhIRo/frxl/4ULFyyLT1wtdnZ2CgwMrHD7i9t6e3tLko4fP65WrVopJSVF33//vdauXWtpYzabVVRUpKNHjyogIECS9fekPMHBwerbt69uvfVW9e/fXw888ECJQF/We7G6zJgxQ1OmTLFs5+TkyNfXt1rvAQAAAGPUqEAmSePGjdOoUaP06quvKjo6WkFBQVbhqDy5ubnq1KmT1Q/mxby8vK64vtzcXEnS5s2b1ahRI6tj9vb2VtvOzs6Wr8tbVKOiddetW9fqmMlkUlFRUcWKv8x9JWnFihXq2rWr1bGLg19l1Knz90xYs9ls2Vf8ObeLOTo6Vmohj4ufvfi84mfPzc3Vk08+qUmTJpU4r3HjxpavL/6elKdjx446evSoPv30U23btk3Dhg3TvffeW+bnxK4Ge3v7Eu8tAAAA3BgMDWR2dnYqLCy02jdgwAA5Oztr+fLl2rJli3bs2FHivN27d2v06NFW28WjUx07dtT69etVv359ubm5VamulJQU/fe//7WEqN27d8vFxUW+vr7y9PSUvb29MjMzraYnlqdFixZydHRUfHy8ZYrbxaqjbqn0Pi1LgwYN5OPjo59++kkjR46s8n0vVhwgs7OzddNNN0lShUeNKlt/sY4dO+rgwYNWnzWsDm5ubgoKClJQUJD+53/+R/3799eff/4pT09PSWW/F8tT1WcFAADAjcPQ30Pm5+enPXv2KCMjQydOnFBRUZFsbGwUHBysGTNmqEWLFiWmhEnShg0btGrVKh0+fNjymbPixRtGjhypm2++WYMGDVJiYqKOHj2qhIQETZo0SceOHatQXfn5+QoJCdHBgwf1ySefKDw8XBMnTlSdOnXk6uqq0NBQTZ48WatXr1Z6erq+/fZbvf7662UuguHg4KDp06dr2rRpWrNmjdLT07V792699dZb1VZ3cZ/u2LFDv/zyi06cOFGhcyIjIxUVFaXXXntNhw8f1v79+xUdHa1XXnmlwve9mL+/v3x9fRUREaEjR45o8+bNWrRoUYXr//7773Xo0CGdOHGi1JG10kyfPl1fffWVJk6cqOTkZB05ckQffvhhiUU9KuOVV15RbGysfvzxRx0+fFgbNmxQw4YNrVasLOu9WB4/Pz/l5uYqPj5eJ06c0Llz56pcKwAAAK5Phgay0NBQ2djYqHXr1vLy8lJmZqYkKSQkRPn5+Ro7dmyp50VGRiouLk6BgYFas2aNYmNj1bp1a0mSk5OTduzYocaNG2vIkCEKCAhQSEiIzp8/X+GRpz59+qhFixbq2bOngoKC9NBDD1ktST579myFhYUpKipKAQEB6t+/vzZv3qymTZuWed2wsDBNnTpVM2fOVEBAgIKCgnT8+PFqq1v6eyGRjIwMNW/evMJTNMeNG6eVK1cqOjpabdu2Va9evRQTE1Pu81xO3bp1LUEmMDBQL730kubMmVOhc8ePH69bb71VnTt3lpeXl5KSkip0XmBgoLZv367Dhw/rrrvuUocOHTRz5kz5+PhU6RkkydXVVQsWLFDnzp11++23KyMjQ5988ollSqZU9nuxPN27d9eECRMUFBQkLy8vLViwoMq1AgAA4PpkMl/8QZ8aIjExUX369FFWVpZlWfhiJpNJGzdu1ODBg6/KvYODg3Xq1Clt2rTpqlwfN46r/V68nJycHLm7u+vFYQ/J4ZLPFAIAAFTFC+9c28/I3+iKf147ffp0uYMrNWpRj7y8PP3xxx+KiIjQ0KFDS4QxAAAAALiRGDpl8VKxsbFq0qSJTp06ddWmb7m4uFz2lZiYeFXueT2bN2/eZfvr/vvvN7q8K3IjPxsAAACuDzVyyuLVlJaWdtljjRo1Knd5+trmzz//1J9//lnqMUdHxxJL/19PrtdnY8oiAACobkxZrF7X7ZTFa6G6l0W/0Xl6elqWeL/R3MjPBgAAgOtDjZqyCAAAAAC1CYEMAAAAAAxCIAMAAAAAgxDIAAAAAMAgBDIAAAAAMAiBDAAAAAAMQiADAAAAAIMQyAAAAADAIAQyAAAAADAIgQwAAAAADGIym81mo4sAUHE5OTlyd3fX6dOn5ebmZnQ5AAAAuERlfl5jhAwAAAAADEIgAwAAAACDEMgAAAAAwCC2RhcAoHKKP/aZk5NjcCUAAAAoTfHPaRVZroNABlxnTp48KUny9fU1uBIAAACU5cyZM3J3dy+zDYEMuM54enpKkjIzM8v9A46ScnJy5Ovrq6ysLFaprAL678rQf1VH310Z+u/K0H9Xpjb2n9ls1pkzZ+Tj41NuWwIZcJ2pU+fvj366u7vXmr/UrgY3Nzf67wrQf1eG/qs6+u7K0H9Xhv67MrWt/yr6H+cs6gEAAAAABiGQAQAAAIBBCGTAdcbe3l7h4eGyt7c3upTrEv13Zei/K0P/VR19d2XovytD/10Z+q9sJnNF1mIEAAAAAFQ7RsgAAAAAwCAEMgAAAAAwCIEMAAAAAAxCIAMAAAAAgxDIgBro3//+t/z8/OTg4KCuXbtq7969ZbbfsGGDWrVqJQcHB7Vt21affPLJNaq0ZqpM/x04cECPPPKI/Pz8ZDKZtHjx4mtXaA1Vmf5bsWKF7rrrLt1000266aabdO+995b7fr2RVabvPvjgA3Xu3FkeHh5ydnZW+/bt9fbbb1/Damueyv7dVywuLk4mk0mDBw++ugXWcJXpv5iYGJlMJquXg4PDNay25qns++/UqVN6+umn5e3tLXt7e7Vs2bJW//tbmf7r3bt3ifefyWTSwIEDr2HFNYgZQI0SFxdntrOzM69atcp84MAB8/jx480eHh7m33//vdT2SUlJZhsbG/OCBQvMBw8eNL/44ovmunXrmvfv33+NK68ZKtt/e/fuNYeGhppjY2PNDRs2NL/66qvXtuAaprL9N2LECPO///1v83fffWdOTU01BwcHm93d3c3Hjh27xpUbr7J99+WXX5o/+OAD88GDB81paWnmxYsXm21sbMxbtmy5xpXXDJXtv2JHjx41N2rUyHzXXXeZBw0adG2KrYEq23/R0dFmNzc3c3Z2tuX122+/XeOqa47K9l9eXp65c+fO5gEDBph37txpPnr0qDkhIcGcnJx8jSuvGSrbfydPnrR67/3www9mGxsbc3R09LUtvIYgkAE1TJcuXcxPP/20ZbuwsNDs4+NjjoqKKrX9sGHDzAMHDrTa17VrV/OTTz55VeusqSrbfxdr0qRJrQ9kV9J/ZrPZfOHCBbOrq6t59erVV6vEGutK+85sNps7dOhgfvHFF69GeTVeVfrvwoUL5u7du5tXrlxpHjNmTK0OZJXtv+joaLO7u/s1qq7mq2z/LV++3NysWTNzfn7+tSqxRrvSv/9effVVs6urqzk3N/dqlVijMWURqEHy8/O1b98+3XvvvZZ9derU0b333qtdu3aVes6uXbus2ktSv379Ltv+RlaV/sP/qY7+O3funAoKCuTp6Xm1yqyRrrTvzGaz4uPjdejQIfXs2fNqllojVbX/Zs2apfr16yskJORalFljVbX/cnNz1aRJE/n6+mrQoEE6cODAtSi3xqlK/3300Ufq1q2bnn76aTVo0EC33Xab5s2bp8LCwmtVdo1RHf92vPXWW3r00Ufl7Ox8tcqs0QhkQA1y4sQJFRYWqkGDBlb7GzRooN9++63Uc3777bdKtb+RVaX/8H+qo/+mT58uHx+fEv9JcKOrat+dPn1aLi4usrOz08CBA/X666+rb9++V7vcGqcq/bdz50699dZbWrFixbUosUarSv/deuutWrVqlT788EO98847KioqUvfu3XXs2LFrUXKNUpX+++mnn/Tee++psLBQn3zyicLCwrRo0SLNmTPnWpRco1zpvx179+7VDz/8oHHjxl2tEms8W6MLAADcGObPn6+4uDglJCTU+sUBKsrV1VXJycnKzc1VfHy8pkyZombNmql3795Gl1ajnTlzRqNGjdKKFSt08803G13Odalbt27q1q2bZbt79+4KCAjQm2++qdmzZxtY2fWhqKhI9evX13/+8x/Z2NioU6dO+uWXX/Tyyy8rPDzc6PKuK2+99Zbatm2rLl26GF2KYQhkQA1y8803y8bGRr///rvV/t9//10NGzYs9ZyGDRtWqv2NrCr9h/9zJf23cOFCzZ8/X9u2bVNgYODVLLNGqmrf1alTR/7+/pKk9u3bKzU1VVFRUbUukFW2/9LT05WRkaEHH3zQsq+oqEiSZGtrq0OHDql58+ZXt+gapDr+7qtbt646dOigtLS0q1FijVaV/vP29lbdunVlY2Nj2RcQEKDffvtN+fn5srOzu6o11yRX8v47e/as4uLiNGvWrKtZYo3HlEWgBrGzs1OnTp0UHx9v2VdUVKT4+Hir/8m8WLdu3azaS9Lnn39+2fY3sqr0H/5PVftvwYIFmj17trZs2aLOnTtfi1JrnOp67xUVFSkvL+9qlFijVbb/WrVqpf379ys5Odnyeuihh3T33XcrOTlZvr6+17J8w1XH+6+wsFD79++Xt7f31SqzxqpK/915551KS0uz/EeAJB0+fFje3t61KoxJV/b+27Bhg/Ly8vTYY49d7TJrNqNXFQFgLS4uzmxvb2+OiYkxHzx40PzEE0+YPTw8LMsRjxo1yvz8889b2iclJZltbW3NCxcuNKempprDw8Nr/bL3lem/vLw883fffWf+7rvvzN7e3ubQ0FDzd999Zz5y5IhRj2Coyvbf/PnzzXZ2dub33nvPagnjM2fOGPUIhqls382bN8+8detWc3p6uvngwYPmhQsXmm1tbc0rVqww6hEMVdn+u1RtX2Wxsv0XGRlp/uyzz8zp6enmffv2mR999FGzg4OD+cCBA0Y9gqEq23+ZmZlmV1dX88SJE82HDh0y/+///q+5fv365jlz5hj1CIaq6p/fHj16mIOCgq51uTUOgQyogV5//XVz48aNzXZ2duYuXbqYd+/ebTnWq1cv85gxY6zav/vuu+aWLVua7ezszG3atDFv3rz5Gldcs1Sm/44ePWqWVOLVq1eva194DVGZ/mvSpEmp/RceHn7tC68BKtN3L7zwgtnf39/s4OBgvummm8zdunUzx8XFGVB1zVHZv/suVtsDmdlcuf579tlnLW0bNGhgHjBggPnbb781oOqao7Lvv6+++srctWtXs729vblZs2bmuXPnmi9cuHCNq645Ktt/P/74o1mSeevWrde40prHZDabzQYNzgEAAABArcZnyAAAAADAIAQyAAAAADAIgQwAAAAADEIgAwAAAACDEMgAAAAAwCAEMgAAAAAwCIEMAAAAAAxCIAMAAAAAgxDIAAAAAMAgBDIAAFCjFRQUGF0CAFw1BDIAAGBly5Yt6tGjhzw8PFSvXj098MADSk9Ptxw/duyYhg8fLk9PTzk7O6tz587as2eP5fjHH3+s22+/XQ4ODrr55pv18MMPW46ZTCZt2rTJ6n4eHh6KiYmRJGVkZMhkMmn9+vXq1auXHBwctHbtWp08eVLDhw9Xo0aN5OTkpLZt2yo2NtbqOkVFRVqwYIH8/f1lb2+vxo0ba+7cuZKke+65RxMnTrRq/8cff8jOzk7x8fHV0W0AUCUEMgAAYOXs2bOaMmWKvvnmG8XHx6tOnTp6+OGHVVRUpNzcXPXq1Uu//PKLPvroI6WkpGjatGkqKiqSJG3evFkPP/ywBgwYoO+++07x8fHq0qVLpWt4/vnn9cwzzyg1NVX9+vXT+fPn1alTJ23evFk//PCDnnjiCY0aNUp79+61nDNjxgzNnz9fYWFhOnjwoNatW6cGDRpIksaNG6d169YpLy/P0v6dd95Ro0aNdM8991xhjwFA1ZnMZrPZ6CIAAEDNdeLECXl5eWn//v366quvFBoaqoyMDHl6epZo2717dzVr1kzvvPNOqdcymUzauHGjBg8ebNnn4eGhxYsXKzg4WBkZGWratKkWL16sZ555psy6HnjgAbVq1UoLFy7UmTNn5OXlpaVLl2rcuHEl2p4/f14+Pj564403NGzYMElSu3btNGTIEIWHh1eiNwCgejFCBgAArBw5ckTDhw9Xs2bN5ObmJj8/P0lSZmamkpOT1aFDh1LDmCQlJyerT58+V1xD586drbYLCws1e/ZstW3bVp6ennJxcdFnn32mzMxMSVJqaqry8vIue28HBweNGjVKq1atkiR9++23+uGHHxQcHHzFtQLAlbA1ugAAAFCzPPjgg2rSpIlWrFghHx8fFRUV6bbbblN+fr4cHR3LPLe84yaTSZdOzilt0Q5nZ2er7ZdffllLlizR4sWL1bZtWzk7O+vZZ59Vfn5+he4r/T1tsX379jp27Jiio6N1zz33qEmTJuWeBwBXEyNkAADA4uTJkzp06JBefPFF9enTRwEBAfrrr78sxwMDA5WcnKw///yz1PMDAwPLXCTDy8tL2dnZlu0jR47o3Llz5daVlJSkQYMG6bHHHlO7du3UrFkzHT582HK8RYsWcnR0LPPebdu2VefOnbVixQqtW7dOjz/+eLn3BYCrjUAGAAAsbrrpJtWrV0//+c9/lJaWpi+++EJTpkyxHB8+fLgaNmyowYMHKykpST/99JPef/997dq1S5IUHh6u2NhYhYeHKzU1Vfv379dLL71kOf+ee+7R0qVL9d133+mbb77RhAkTVLdu3XLratGihT7//HN99dVXSk1N1ZNPPqnff//dctzBwUHTp0/XtGnTtGbNGqWnp2v37t166623rK4zbtw4zZ8/X2az2Wr1RwAwCoEMAABY1KlTR3Fxcdq3b59uu+02TZ48WS+//LLluJ2dnbZu3ar69etrwIABatu2rebPny8bGxtJUu/evbVhwwZ99NFHat++ve655x6rlRAXLVokX19f3XXXXRoxYoRCQ0Pl5ORUbl0vvviiOnbsqH79+ql3796WUHixsLAwTZ06VTNnzlRAQICCgoJ0/PhxqzbDhw+Xra2thg8fLgcHhyvoKQCoHqyyCAAAao2MjAw1b95cX3/9tTp27Gh0OQBAIAMAADe+goICnTx5UqGhoTp69KiSkpKMLgkAJDFlEQAA1AJJSUny9vbW119/rTfeeMPocgDAghEyAAAAADAII2QAAAAAYBACGQAAAAAYhEAGAAAAAAYhkAEAAACAQQhkAAAAAGAQAhkAAAAAGIRABgAAAAAGIZABAAAAgEEIZAAAAABgkP8HFtR0MH+GscsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 800x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "plt.figure(figsize=(8, 4))\n",
    "sns.barplot(y='key', x='accuracy', data=metrics_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9fc0c11c-be15-47dc-88bf-31b0192b4622",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1058.563616,
   "end_time": "2024-11-23T14:46:37.625874",
   "environment_variables": {},
   "exception": null,
   "input_path": "13_contextual_embeddings.ipynb",
   "output_path": "run_13_contextual_embeddings.ipynb",
   "parameters": {},
   "start_time": "2024-11-23T14:28:59.062258",
   "version": "2.6.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "0cd8c168767249f2a5fa412173f6e751": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_5ce1d1d9d86c40d9839877ff95734491",
       "max": 100,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_231702cf4d79477f9d5548665a1b18fe",
       "tabbable": null,
       "tooltip": null,
       "value": 100
      }
     },
     "2133bb8d85d34b8db112b4408ad60320": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "231702cf4d79477f9d5548665a1b18fe": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "23b1ad9c0f9c46c888da66e85c90eb84": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "24e6eadc3dc940ecabf30dd1a3c6d1f3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_fa4bddf2c33241b5bf918054518f128f",
       "max": 52,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_edc33e82be8f41eba6a18a0ef074ab7a",
       "tabbable": null,
       "tooltip": null,
       "value": 52
      }
     },
     "2f60367b1c8941e2bf71661c33969ae8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3865f25c78aa46f29a25d807205281c3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3d0b06deaa654b989eece8cde06fa0f8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "3f8ceda83287475b97608e42f5f6782f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4881e496f1c84fe29ce9ebebaddfb3c2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_bd096d5d219a467786a85cfe1613fedd",
        "IPY_MODEL_24e6eadc3dc940ecabf30dd1a3c6d1f3",
        "IPY_MODEL_bc2b8104b4244d8cacedeb95e800d91c"
       ],
       "layout": "IPY_MODEL_6b9a8e43c1c342dba500a14e7149b600",
       "tabbable": null,
       "tooltip": null
      }
     },
     "5ce1d1d9d86c40d9839877ff95734491": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5ddb08be5cc64c9ab40a1d62a21763a5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_86283159049d48b1adcfb2de2d404d4d",
       "placeholder": "​",
       "style": "IPY_MODEL_2133bb8d85d34b8db112b4408ad60320",
       "tabbable": null,
       "tooltip": null,
       "value": " 100/100 [08:34&lt;00:00, 10.01s/it]"
      }
     },
     "5ef9d83ccad1471f85335900a24a8553": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6b9a8e43c1c342dba500a14e7149b600": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "816a079a8c804fbfa9b9a74f941abea8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_bcc69ec5db1b4aab977807284c9290e7",
        "IPY_MODEL_0cd8c168767249f2a5fa412173f6e751",
        "IPY_MODEL_5ddb08be5cc64c9ab40a1d62a21763a5"
       ],
       "layout": "IPY_MODEL_d1178c6858284f788a80b5f2a14fd0b7",
       "tabbable": null,
       "tooltip": null
      }
     },
     "86283159049d48b1adcfb2de2d404d4d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8ff8262c56604119883f4a5f13bb74ab": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_5ef9d83ccad1471f85335900a24a8553",
       "placeholder": "​",
       "style": "IPY_MODEL_e89e77133c344fc48c1d62f5a607ec93",
       "tabbable": null,
       "tooltip": null,
       "value": " 8/8 [00:18&lt;00:00,  2.27s/it]"
      }
     },
     "9189a076554543aaa6f5ee04e40dbe1b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "988e6697a2af486fadeaf0b84347b565": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_e1aae4c55cb64f379e74f15357275628",
        "IPY_MODEL_fd9e23198ca1489a9773fda3510bf857",
        "IPY_MODEL_8ff8262c56604119883f4a5f13bb74ab"
       ],
       "layout": "IPY_MODEL_d2ee15001d2244529f7e47d3333c0f8e",
       "tabbable": null,
       "tooltip": null
      }
     },
     "9fc7d91f94a94933bde5ba80e64587de": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "a7d240a289084bdfba4724c0efd5ab07": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "bc2b8104b4244d8cacedeb95e800d91c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_2f60367b1c8941e2bf71661c33969ae8",
       "placeholder": "​",
       "style": "IPY_MODEL_9fc7d91f94a94933bde5ba80e64587de",
       "tabbable": null,
       "tooltip": null,
       "value": " 52/52 [04:26&lt;00:00,  4.22s/it]"
      }
     },
     "bcc69ec5db1b4aab977807284c9290e7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_3f8ceda83287475b97608e42f5f6782f",
       "placeholder": "​",
       "style": "IPY_MODEL_3d0b06deaa654b989eece8cde06fa0f8",
       "tabbable": null,
       "tooltip": null,
       "value": "100%"
      }
     },
     "bd096d5d219a467786a85cfe1613fedd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_3865f25c78aa46f29a25d807205281c3",
       "placeholder": "​",
       "style": "IPY_MODEL_9189a076554543aaa6f5ee04e40dbe1b",
       "tabbable": null,
       "tooltip": null,
       "value": "100%"
      }
     },
     "cc3ed8dc4a5c43aca7b62d904865b2fa": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "cf68b6fe24964ce792aa63827489cb97": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "d1178c6858284f788a80b5f2a14fd0b7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d2ee15001d2244529f7e47d3333c0f8e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e1aae4c55cb64f379e74f15357275628": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_23b1ad9c0f9c46c888da66e85c90eb84",
       "placeholder": "​",
       "style": "IPY_MODEL_cf68b6fe24964ce792aa63827489cb97",
       "tabbable": null,
       "tooltip": null,
       "value": "100%"
      }
     },
     "e89e77133c344fc48c1d62f5a607ec93": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "edc33e82be8f41eba6a18a0ef074ab7a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "fa4bddf2c33241b5bf918054518f128f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "fd9e23198ca1489a9773fda3510bf857": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_cc3ed8dc4a5c43aca7b62d904865b2fa",
       "max": 8,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_a7d240a289084bdfba4724c0efd5ab07",
       "tabbable": null,
       "tooltip": null,
       "value": 8
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
